Index: robots.txt
===================================================================
RCS file: /cvs/drupal/drupal/robots.txt,v
retrieving revision 1.14
diff -u -9 -p -r1.14 robots.txt
--- robots.txt	11 Sep 2009 04:17:07 -0000	1.14
+++ robots.txt	20 Sep 2009 19:50:03 -0000
@@ -15,23 +15,20 @@
 # http://www.robotstxt.org/wc/robots.html
 #
 # For syntax checking, see:
 # http://www.sxw.org.uk/computing/robots/check.html
 
 User-agent: *
 Crawl-delay: 10
 # Directories
 Disallow: /includes/
-Disallow: /misc/
-Disallow: /modules/
 Disallow: /profiles/
 Disallow: /scripts/
-Disallow: /themes/
 # Files
 Disallow: /CHANGELOG.txt
 Disallow: /cron.php
 Disallow: /INSTALL.mysql.txt
 Disallow: /INSTALL.pgsql.txt
 Disallow: /install.php
 Disallow: /INSTALL.txt
 Disallow: /LICENSE.txt
 Disallow: /MAINTAINERS.txt