Index: .htaccess =================================================================== RCS file: /cvs/drupal/drupal/.htaccess,v retrieving revision 1.111 diff -u -r1.111 .htaccess --- .htaccess 23 Nov 2010 02:59:05 -0000 1.111 +++ .htaccess 14 Feb 2011 19:09:57 -0000 @@ -106,6 +106,9 @@ # uncomment the following line: # RewriteBase / + # Pass requests for robots.txt over to index.php, even if robots.txt already exists. + RewriteRule ^robots.txt index.php?/$1$2 [last,qsappend] + # Pass all requests not referring directly to files in the filesystem to # index.php. Clean URLs are handled in drupal_environment_initialize(). RewriteCond %{REQUEST_FILENAME} !-f Index: includes/common.inc =================================================================== RCS file: /cvs/drupal/drupal/includes/common.inc,v retrieving revision 1.1287 diff -u -r1.1287 common.inc --- includes/common.inc 9 Feb 2011 02:23:52 -0000 1.1287 +++ includes/common.inc 14 Feb 2011 19:10:24 -0000 @@ -219,6 +219,22 @@ return $profile; } +/** + * Gets the contents of the robots.txt file. + * + * @see hook_robotstxt() + * @see hook_robotstxt_alter() + */ +function drupal_get_robotstxt() { + $cache = &drupal_static(__FUNCTION__, ''); + if (empty($cache)) { + $cache = module_invoke_all('robotstxt'); + drupal_alter('robotstxt', $cache); + $cache = implode("\n", $cache); + } + return $cache; +} + /** * Set the breadcrumb trail for the current page. @@ -2541,6 +2557,116 @@ } /** + * Package and send the result of a page callback to the browser as text. + * + * @param $page_callback_result + * The result of a page callback. Can be one of: + * - NULL: to indicate no content. + * - An integer menu status constant: to indicate an error condition. + * - A string of HTML content. + * - A renderable array of content. + * + * @see drupal_deliver_page() + */ +function drupal_deliver_txt_page($page_callback_result) { + // Emit the correct charset HTTP header, but not if the page callback + // result is NULL, since that likely indicates that it printed something + // in which case, no further headers may be sent, and not if code running + // for this page request has already set the content type header. + if (isset($page_callback_result) && is_null(drupal_get_http_header('Content-Type'))) { + drupal_add_http_header('Content-Type', 'text/plain; charset=utf-8'); + } + + // Menu status constants are integers; page content is a string or array. + if (is_int($page_callback_result)) { + // @todo: Break these up into separate functions? + switch ($page_callback_result) { + case MENU_NOT_FOUND: + // Print a 404 page. + drupal_add_http_header('Status', '404 Not Found'); + + watchdog('page not found', check_plain($_GET['q']), NULL, WATCHDOG_WARNING); + + // Keep old path for reference, and to allow forms to redirect to it. + if (!isset($_GET['destination'])) { + $_GET['destination'] = $_GET['q']; + } + + $path = drupal_get_normal_path(variable_get('site_404', '')); + if ($path && $path != $_GET['q']) { + // Custom 404 handler. Set the active item in case there are tabs to + // display, or other dependencies on the path. + menu_set_active_item($path); + $return = menu_execute_active_handler($path, FALSE); + } + + if (empty($return) || $return == MENU_NOT_FOUND || $return == MENU_ACCESS_DENIED) { + // Standard 404 handler. + drupal_set_title(t('Page not found')); + $return = t('The requested page could not be found.'); + } + + drupal_set_page_content($return); + $page = element_info('page'); + $page['#theme_wrappers'] = array('txt'); + print drupal_render_page($page); + break; + + case MENU_ACCESS_DENIED: + // Print a 403 page. + drupal_add_http_header('Status', '403 Forbidden'); + watchdog('access denied', check_plain($_GET['q']), NULL, WATCHDOG_WARNING); + + // Keep old path for reference, and to allow forms to redirect to it. + if (!isset($_GET['destination'])) { + $_GET['destination'] = $_GET['q']; + } + + $path = drupal_get_normal_path(variable_get('site_403', '')); + if ($path && $path != $_GET['q']) { + // Custom 403 handler. Set the active item in case there are tabs to + // display or other dependencies on the path. + menu_set_active_item($path); + $return = menu_execute_active_handler($path, FALSE); + } + + if (empty($return) || $return == MENU_NOT_FOUND || $return == MENU_ACCESS_DENIED) { + // Standard 403 handler. + drupal_set_title(t('Access denied')); + $return = t('You are not authorized to access this page.'); + } + + drupal_set_page_content($return); + $page = element_info('page'); + $page['#theme_wrappers'] = array('txt'); + print drupal_render_page($page); + break; + + case MENU_SITE_OFFLINE: + // Print a 503 page. + drupal_maintenance_theme(); + drupal_add_http_header('Status', '503 Service unavailable'); + drupal_set_title(t('Site under maintenance')); + $return = variable_get('maintenance_mode_message', t('@site is currently under maintenance. We should be back shortly. Thank you for your patience.', array('@site' => variable_get('site_name', 'Drupal')))); + + drupal_set_page_content($return); + $page = element_info('page'); + $page['#theme_wrappers'] = array('txt'); + print drupal_render_page($page); + break; + } + } + elseif (isset($page_callback_result)) { + // Print anything besides a menu constant, assuming it's not NULL or + // undefined. + print drupal_render_page($page_callback_result); + } + + // Perform end-of-request tasks. + drupal_page_footer(); +} + +/** * Perform end-of-request tasks. * * This function sets the page cache if appropriate, and allows modules to Index: modules/system/system.api.php =================================================================== RCS file: /cvs/drupal/drupal/modules/system/system.api.php,v retrieving revision 1.228 diff -u -r1.228 system.api.php --- modules/system/system.api.php 9 Feb 2011 02:26:04 -0000 1.228 +++ modules/system/system.api.php 14 Feb 2011 19:11:06 -0000 @@ -3719,6 +3719,28 @@ } /** + * Appends content to the robots.txt. + * + * @return An array of strings to append to the end of the robots.txt file. + */ +function hook_robotstxt() { + return array('Disallow: /tmp/'); +} + +/** + * Appends content to the robots.txt. + * + * @return An array of strings to append to the end of the robots.txt file. + */ +function hook_robotstxt_alter(&$robotstxt) { + foreach ($robotstxt as $index => $line) { + if ($line == 'Disallow: /tmp/') { + $robotstxt[$index] = 'Disallow: /temp/'; + } + } +} + +/** * Define additional date types. * * Next to the 'long', 'medium' and 'short' date types defined in core, any Index: modules/system/system.module =================================================================== RCS file: /cvs/drupal/drupal/modules/system/system.module,v retrieving revision 1.1003 diff -u -r1.1003 system.module --- modules/system/system.module 4 Jan 2011 00:56:23 -0000 1.1003 +++ modules/system/system.module 14 Feb 2011 19:11:26 -0000 @@ -1056,6 +1057,14 @@ 'type' => MENU_CALLBACK, 'file' => 'system.admin.inc', ); + + // Search engine control. + $items['robots.txt'] = array( + 'page callback' => 'drupal_get_robotstxt', + 'access callback' => TRUE, + 'type' => MENU_CALLBACK, + 'delivery callback' => 'drupal_deliver_txt_page', + ); return $items; } @@ -3899,6 +3908,24 @@ } /** + * Implements hook_robotstxt(). + */ +function system_robotstxt() { + // Cache the robots.txt content from the file system. + $robotstxt = &drupal_static(__FUNCTION__, array()); + if (empty($robotstxt)) { + if ($cache = cache_get(__FUNCTION__)) { + $robotstxt = $cache->data; + } + else { + $robotstxt = file(realpath('robots.txt'), FILE_IGNORE_NEW_LINES); + cache_set(__FUNCTION__, $robotstxt); + } + } + return $robotstxt; +} + +/** * Returns HTML for a confirmation form. * * By default this does not alter the appearance of a form at all, Index: modules/system/txt.tpl.php =================================================================== RCS file: modules/system/txt.tpl.php diff -N modules/system/txt.tpl.php --- /dev/null 1 Jan 1970 00:00:00 -0000 +++ modules/system/txt.tpl.php 1 Jan 1970 00:00:00 -0000 @@ -0,0 +1,25 @@ +