robots.txt

Same filename in other branches
  1. 9 core/tests/Drupal/Tests/Composer/Plugin/Scaffold/fixtures/drupal-assets-fixture/assets/robots.txt
  2. 9 core/assets/scaffold/files/robots.txt
  3. 9 robots.txt
  4. 8.9.x core/tests/Drupal/Tests/Composer/Plugin/Scaffold/fixtures/drupal-assets-fixture/assets/robots.txt
  5. 8.9.x core/assets/scaffold/files/robots.txt
  6. 8.9.x robots.txt
  7. 10 core/tests/Drupal/Tests/Composer/Plugin/Scaffold/fixtures/drupal-assets-fixture/assets/robots.txt
  8. 10 core/assets/scaffold/files/robots.txt
  9. 10 robots.txt
  10. 11.x core/tests/Drupal/Tests/Composer/Plugin/Scaffold/fixtures/drupal-assets-fixture/assets/robots.txt
  11. 11.x core/assets/scaffold/files/robots.txt
  12. 11.x robots.txt
#
# robots.txt
#
# This file is to prevent the crawling and indexing of certain parts
# of your site by web crawlers and spiders run by sites like Yahoo!
# and Google. By telling these "robots" where not to go on your site,
# you save bandwidth and server resources.
#
# This file will be ignored unless it is at the root of your host:
# Used:    http://example.com/robots.txt
# Ignored: http://example.com/site/robots.txt
#
# For more information about the robots.txt standard, see:
# http://www.robotstxt.org/robotstxt.html

User-agent: *
Crawl-delay: 10
# CSS, JS, Images
Allow: /misc/*.css$
Allow: /misc/*.css?
Allow: /misc/*.js$
Allow: /misc/*.js?
Allow: /misc/*.gif
Allow: /misc/*.jpg
Allow: /misc/*.jpeg
Allow: /misc/*.png
Allow: /modules/*.css$
Allow: /modules/*.css?
Allow: /modules/*.js$
Allow: /modules/*.js?
Allow: /modules/*.gif
Allow: /modules/*.jpg
Allow: /modules/*.jpeg
Allow: /modules/*.png
Allow: /profiles/*.css$
Allow: /profiles/*.css?
Allow: /profiles/*.js$
Allow: /profiles/*.js?
Allow: /profiles/*.gif
Allow: /profiles/*.jpg
Allow: /profiles/*.jpeg
Allow: /profiles/*.png
Allow: /themes/*.css$
Allow: /themes/*.css?
Allow: /themes/*.js$
Allow: /themes/*.js?
Allow: /themes/*.gif
Allow: /themes/*.jpg
Allow: /themes/*.jpeg
Allow: /themes/*.png
# Directories
Disallow: /includes/
Disallow: /misc/
Disallow: /modules/
Disallow: /profiles/
Disallow: /scripts/
Disallow: /themes/
# Files
Disallow: /CHANGELOG.txt
Disallow: /cron.php
Disallow: /INSTALL.mysql.txt
Disallow: /INSTALL.pgsql.txt
Disallow: /INSTALL.sqlite.txt
Disallow: /install.php
Disallow: /INSTALL.txt
Disallow: /LICENSE.txt
Disallow: /MAINTAINERS.txt
Disallow: /update.php
Disallow: /UPGRADE.txt
Disallow: /xmlrpc.php
# Paths (clean URLs)
Disallow: /admin/
Disallow: /comment/reply/
Disallow: /filter/tips/
Disallow: /node/add/
Disallow: /search/
Disallow: /user/register/
Disallow: /user/password/
Disallow: /user/login/
Disallow: /user/logout/
# Paths (no clean URLs)
Disallow: /?q=admin/
Disallow: /?q=comment/reply/
Disallow: /?q=filter/tips/
Disallow: /?q=node/add/
Disallow: /?q=search/
Disallow: /?q=user/password/
Disallow: /?q=user/register/
Disallow: /?q=user/login/
Disallow: /?q=user/logout/

File

./robots.txt

View source 
  1. #

  2. # robots.txt

  3. #

  4. # This file is to prevent the crawling and indexing of certain parts

  5. # of your site by web crawlers and spiders run by sites like Yahoo!

  6. # and Google. By telling these "robots" where not to go on your site,

  7. # you save bandwidth and server resources.

  8. #

  9. # This file will be ignored unless it is at the root of your host:

  10. # Used:    http://example.com/robots.txt

  11. # Ignored: http://example.com/site/robots.txt

  12. #

  13. # For more information about the robots.txt standard, see:

  14. # http://www.robotstxt.org/robotstxt.html

  15. 

  16. User-agent: *

  17. Crawl-delay: 10

  18. # CSS, JS, Images

  19. Allow: /misc/*.css$

  20. Allow: /misc/*.css?

  21. Allow: /misc/*.js$

  22. Allow: /misc/*.js?

  23. Allow: /misc/*.gif

  24. Allow: /misc/*.jpg

  25. Allow: /misc/*.jpeg

  26. Allow: /misc/*.png

  27. Allow: /modules/*.css$

  28. Allow: /modules/*.css?

  29. Allow: /modules/*.js$

  30. Allow: /modules/*.js?

  31. Allow: /modules/*.gif

  32. Allow: /modules/*.jpg

  33. Allow: /modules/*.jpeg

  34. Allow: /modules/*.png

  35. Allow: /profiles/*.css$

  36. Allow: /profiles/*.css?

  37. Allow: /profiles/*.js$

  38. Allow: /profiles/*.js?

  39. Allow: /profiles/*.gif

  40. Allow: /profiles/*.jpg

  41. Allow: /profiles/*.jpeg

  42. Allow: /profiles/*.png

  43. Allow: /themes/*.css$

  44. Allow: /themes/*.css?

  45. Allow: /themes/*.js$

  46. Allow: /themes/*.js?

  47. Allow: /themes/*.gif

  48. Allow: /themes/*.jpg

  49. Allow: /themes/*.jpeg

  50. Allow: /themes/*.png

  51. # Directories

  52. Disallow: /includes/

  53. Disallow: /misc/

  54. Disallow: /modules/

  55. Disallow: /profiles/

  56. Disallow: /scripts/

  57. Disallow: /themes/

  58. # Files

  59. Disallow: /CHANGELOG.txt

  60. Disallow: /cron.php

  61. Disallow: /INSTALL.mysql.txt

  62. Disallow: /INSTALL.pgsql.txt

  63. Disallow: /INSTALL.sqlite.txt

  64. Disallow: /install.php

  65. Disallow: /INSTALL.txt

  66. Disallow: /LICENSE.txt

  67. Disallow: /MAINTAINERS.txt

  68. Disallow: /update.php

  69. Disallow: /UPGRADE.txt

  70. Disallow: /xmlrpc.php

  71. # Paths (clean URLs)

  72. Disallow: /admin/

  73. Disallow: /comment/reply/

  74. Disallow: /filter/tips/

  75. Disallow: /node/add/

  76. Disallow: /search/

  77. Disallow: /user/register/

  78. Disallow: /user/password/

  79. Disallow: /user/login/

  80. Disallow: /user/logout/

  81. # Paths (no clean URLs)

  82. Disallow: /?q=admin/

  83. Disallow: /?q=comment/reply/

  84. Disallow: /?q=filter/tips/

  85. Disallow: /?q=node/add/

  86. Disallow: /?q=search/

  87. Disallow: /?q=user/password/

  88. Disallow: /?q=user/register/

  89. Disallow: /?q=user/login/

  90. Disallow: /?q=user/logout/

Buggy or inaccurate documentation? Please file an issue. Need support? Need help programming? Connect with the Drupal community.