Просмотр файла htmly-2.9.8/robots.txt

Конкурс дизайнеров на 15000 рублей
контркультура падонки бабруйск
Размер файла: 1.13Kb 
  1. #
  2. # robots.txt
  3. #
  4. # This file is to prevent the crawling and indexing of certain parts
  5. # of your site by web crawlers and spiders run by sites like Yahoo!
  6. # and Google. By telling these "robots" where not to go on your site,
  7. # you save bandwidth and server resources.
  8. #
  9. # This file will be ignored unless it is at the root of your host:
  10. # Used:    http://example.com/robots.txt
  11. # Ignored: http://example.com/site/robots.txt
  12. #
  13. # For more information about the robots.txt standard, see:
  14. # http://www.robotstxt.org/wc/robots.html
  15. #
  16. # For syntax checking, see:
  17. # http://www.sxw.org.uk/computing/robots/check.html
  18.  
  19. User-agent: *
  20.  
  21. # Disallow directories
  22. Disallow: /config/
  23. Disallow: /system/
  24. Disallow: /themes/
  25. Disallow: /vendor/
  26. Disallow: /cache/
  27.  
  28. # Disallow files
  29. Disallow: /changelog.txt
  30. Disallow: /composer.json
  31. Disallow: /composer.lock
  32. Disallow: /composer.phar
  33.  
  34. # Disallow paths
  35. Disallow: /search/
  36. Disallow: /admin/
  37.  
  38. # Allow themes
  39. Allow: /themes/*/css/
  40. Allow: /themes/*/images/
  41. Allow: /themes/*/img/
  42. Allow: /themes/*/js/
  43. Allow: /themes/*/fonts/
  44.  
  45. # Allow content images
  46. Allow: /content/images/*.jpg
  47. Allow: /content/images/*.png
  48. Allow: /content/images/*.gif