Index: includes/linkchecker.admin.inc
===================================================================
RCS file: /cvs/drupal-contrib/contributions/modules/linkchecker/includes/linkchecker.admin.inc,v
retrieving revision 1.1.2.24
diff -u -p -r1.1.2.24 linkchecker.admin.inc
--- includes/linkchecker.admin.inc Base (1.1.2.24)
+++ includes/linkchecker.admin.inc Locally Modified (Based On 1.1.2.24)
@@ -154,6 +154,18 @@
'#description' => t('By default this list contains the domain names reserved for use in documentation and not available for registration. See RFC 2606, Section 3 for more information. URLs on this list are still extracted, but the link setting Check link status becomes automatically disabled to prevent false alarms. If you change this list you need to clear all link data and re-analyze your content. Otherwise this setting will only affect new links added after the configuration change.', array('@rfc-2606' => 'http://www.rfc-editor.org/rfc/rfc2606.txt')),
'#wysiwyg' => FALSE,
);
+ $form['check']['linkchecker_max_connections'] = array(
+ '#default_value' => variable_get('linkchecker_max_connections', LINKCHECKER_DEFAULT_MAX_CONNECTIONS),
+ '#type' => 'select',
+ '#title' => t('Maximum number of simultaneous connections'),
+ '#description' => t('Setting this number higher can help speed up the checking process during cron runs. Set lower if Drupal times out or uses too much memory during Cron calls.'),
+ '#options' => drupal_map_assoc(array(1, 5, 10, 20, 50)),
+ );
+ // Disable max_connections form when CURL library is not present
+ if (!function_exists('curl_init')) {
+ $form['check']['linkchecker_max_connections']['#disabled'] = TRUE;
+ $form['check']['linkchecker_max_connections']['#description'] .= '' . t('Disabled because CURL library is not present') . '
';
+ }
$form['error'] = array(
'#type' => 'fieldset',
Index: linkchecker.module
===================================================================
RCS file: /cvs/drupal-contrib/contributions/modules/linkchecker/linkchecker.module,v
retrieving revision 1.7.2.138
diff -u -p -r1.7.2.138 linkchecker.module
--- linkchecker.module Base (1.7.2.138)
+++ linkchecker.module Locally Modified (Based On 1.7.2.138)
@@ -15,6 +15,12 @@
define('LINKCHECKER_SCAN_MAX_LINKS_PER_RUN', '100');
/**
+ * Defines the default maximum number of simultaneous connections to use when
+ * CURL library is available.
+ */
+define('LINKCHECKER_DEFAULT_MAX_CONNECTIONS', 10);
+
+/**
* A list of domain names reserved for use in documentation and not available
* for registration. See RFC 2606, Section 3 for more information.
*/
@@ -121,10 +127,6 @@
_linkchecker_cleanup_links();
variable_set('linkchecker_cleanup_links_last', time());
}
-
- // TODO: Implement cURL support.
- //$has_curl = function_exists('curl_init');
-
// TODO: Remove some confusion about the max links that can be checked per
// cron run and guess that 2 link can be checked per second what is
// nevertheless uncommon. But we can use the max_execution_time to calculate
@@ -137,20 +139,187 @@
$check_links_interval = variable_get('linkchecker_check_links_interval', 2419200);
$useragent = variable_get('linkchecker_check_useragent', 'Drupal (+http://drupal.org/)');
+ // Check if CURL available for simultaneous checking
+ $use_curl = function_exists('curl_init') && (variable_get('linkchecker_max_connections', LINKCHECKER_DEFAULT_MAX_CONNECTIONS) > 1);
+ $links_buffer = array();
+
// Get URLs for checking.
$result = db_query_range("SELECT * FROM {linkchecker_links} WHERE last_checked < %d AND status = %d ORDER BY last_checked, lid ASC", time() - $check_links_interval, 1, 0, $check_links_max_per_cron_run);
while ($link = db_fetch_object($result)) {
- // Fetch URL.
+ if (!$use_curl) {
+ // Fetch single URL.
$response = drupal_http_request($link->url, array('User-Agent' => 'User-Agent: ' . $useragent), $link->method, NULL, 1);
_linkchecker_status_handling($link, $response);
-
+ }
+ else {
+ // Fetch N URLs simultaneously and process.
+ $links_buffer[] = $link;
+ if (sizeof($links_buffer) == variable_get('linkchecker_max_connections', LINKCHECKER_DEFAULT_MAX_CONNECTIONS)) {
+ _linkchecker_cron_multi($links_buffer, array('User-Agent' => 'User-Agent: ' . $useragent), 1);
+ // Empty the buffer
+ $links_buffer = array();
+ }
+ }
if ((timer_read('page') / 1000) > ($max_execution_time / 2)) {
- break; // Stop once we have used over half of the maximum execution time.
+ return; // Stop once we have used over half of the maximum execution time.
}
}
+ // If any links remain in buffer, process them
+ if (sizeof($links_buffer) > 0) {
+ _linkchecker_cron_multi($links_buffer, array('User-Agent' => 'User-Agent: ' . $useragent), 1);
}
+}
/**
+ * Pipelined fetch from the Web. Requires PHP CURL library.
+ * Uses code from drupal_http_request() to parse each result.
+ * @param $requests
+ * Array of {linkchecker_link} table records to process.
+ * @param $headers
+ * Extra headers to add to each request.
+ * @param $retry
+ * Maximum number of retries. See drupal_http_request()
+ *
+ * See: http://www.phpied.com/simultaneuos-http-requests-in-php-with-curl/
+ */
+function _linkchecker_cron_multi($requests, $headers = array(), $retry = 3) {
+ // If no requests
+ if (!$requests) {
+ return;
+ }
+
+ // Initialize array of curl handles
+ $curl_handles = array();
+
+ // Headers
+ $defaults = array(
+ // RFC 2616: "non-standard ports MUST, default ports MAY be included".
+ // We don't add the port to prevent from breaking rewrite rules checking the
+ // host that do not take into account the port number.
+ #'Host' => "Host: $host",
+ 'User-Agent' => 'User-Agent: Drupal (+http://drupal.org/)',
+ #'User-Agent' => 'User-Agent: Mozilla/5.0 (Windows; U; Windows NT 5.2; rv:1.9.2) Gecko/20100101 Firefox/3.6',
+ #'Content-Length' => 'Content-Length: '. strlen($data)
+ );
+ foreach ($headers as $header => $value) {
+ $defaults[$header] = $header . ': ' . $value;
+ }
+
+ // multi handle
+ $multi_handle = curl_multi_init();
+
+ // loop through $data and create curl handles
+ // then add them to the multi-handle
+ foreach ($requests as $index => $request) {
+ $curl_handles[$index] = curl_init();
+ curl_setopt($curl_handles[$index], CURLOPT_URL, $request->url);
+ curl_setopt($curl_handles[$index], CURLOPT_HEADER, TRUE);
+ curl_setopt($curl_handles[$index], CURLOPT_RETURNTRANSFER, 1);
+ curl_setopt($curl_handles[$index], CURLOPT_FOLLOWLOCATION, FALSE);
+ curl_setopt($curl_handles[$index], CURLOPT_BINARYTRANSFER, TRUE);
+ curl_setopt($curl_handles[$index], CURLOPT_FAILONERROR, FALSE);
+ curl_setopt($curl_handles[$index], CURLOPT_TIMEOUT, 15);
+
+ // GET or HEAD?
+ if ($request->method == 'HEAD') {
+ curl_setopt($curl_handles[$index], CURLOPT_NOBODY, TRUE);
+ }
+ else {
+ curl_setopt($curl_handles[$index], CURLOPT_HTTPGET, TRUE);
+ }
+
+ // post?
+ if (!empty($request->data)) {
+ curl_setopt($curl_handles[$index], CURLOPT_POST, 1);
+ curl_setopt($curl_handles[$index], CURLOPT_POSTFIELDS, $request->data);
+ }
+
+ // Headers
+ curl_setopt($curl_handles[$index], CURLOPT_HTTPHEADER, $defaults);
+
+ curl_multi_add_handle($multi_handle, $curl_handles[$index]);
+ }
+
+ // execute the handles
+ $running = null;
+ do {
+ curl_multi_exec($multi_handle, $running);
+ } while ($running > 0);
+
+ // Get individual results and remove handles
+ foreach ($curl_handles as $index => $c) {
+ $result = new stdClass();
+ $response = curl_multi_getcontent($c);
+ // Check for errors
+ if (!$response) {
+ // Timeout or error
+ // TODO maybe log to watchdog?
+ $result->code = curl_errno($c);
+ $result->error = curl_error($c);
+ }
+ else {
+ // Parse response.
+ list($split, $result->data) = explode("\r\n\r\n", $response, 2);
+ $split = preg_split("/\r\n|\n|\r/", $split);
+
+ list($protocol, $code, $text) = explode(' ', trim(array_shift($split)), 3);
+ $result->headers = array();
+
+ // Parse headers.
+ while ($line = trim(array_shift($split))) {
+ list($header, $value) = explode(':', $line, 2);
+ if (isset($result->headers[$header]) && $header == 'Set-Cookie') {
+ // RFC 2109: the Set-Cookie response header comprises the token Set-
+ // Cookie:, followed by a comma-separated list of one or more cookies.
+ $result->headers[$header] .= ','. trim($value);
+ }
+ else {
+ $result->headers[$header] = trim($value);
+ }
+ }
+ $responses = array(
+ 100 => 'Continue', 101 => 'Switching Protocols',
+ 200 => 'OK', 201 => 'Created', 202 => 'Accepted', 203 => 'Non-Authoritative Information', 204 => 'No Content', 205 => 'Reset Content', 206 => 'Partial Content',
+ 300 => 'Multiple Choices', 301 => 'Moved Permanently', 302 => 'Found', 303 => 'See Other', 304 => 'Not Modified', 305 => 'Use Proxy', 307 => 'Temporary Redirect',
+ 400 => 'Bad Request', 401 => 'Unauthorized', 402 => 'Payment Required', 403 => 'Forbidden', 404 => 'Not Found', 405 => 'Method Not Allowed', 406 => 'Not Acceptable', 407 => 'Proxy Authentication Required', 408 => 'Request Time-out', 409 => 'Conflict', 410 => 'Gone', 411 => 'Length Required', 412 => 'Precondition Failed', 413 => 'Request Entity Too Large', 414 => 'Request-URI Too Large', 415 => 'Unsupported Media Type', 416 => 'Requested range not satisfiable', 417 => 'Expectation Failed',
+ 500 => 'Internal Server Error', 501 => 'Not Implemented', 502 => 'Bad Gateway', 503 => 'Service Unavailable', 504 => 'Gateway Time-out', 505 => 'HTTP Version not supported'
+ );
+ // RFC 2616 states that all unknown HTTP codes must be treated the same as the
+ // base code in their class.
+ if (!isset($responses[$code])) {
+ $code = floor($code / 100) * 100;
+ }
+ switch ($code) {
+ case 200: // OK
+ case 304: // Not modified
+ break;
+ case 301: // Moved permanently
+ case 302: // Moved temporarily
+ case 307: // Moved temporarily
+ $location = $result->headers['Location'];
+
+ // If retry is set, fall back to drupal_http_request()
+ if ($retry) {
+ $result = drupal_http_request($result->headers['Location'], $headers, $method, $data, --$retry);
+ $result->redirect_code = $result->code;
+ }
+ $result->redirect_url = $location;
+
+ break;
+ default:
+ $result->error = $text;
+ }
+ $result->code = $code;
+ }
+ curl_multi_remove_handle($multi_handle, $c);
+ _linkchecker_status_handling($requests[$index], $result);
+ }
+
+ // all done
+ curl_multi_close($multi_handle);
+}
+
+/**
* Status code handling.
*
* @param $link