diff --git a/httprl.admin.inc b/httprl.admin.inc index a835754..7167454 100644 --- a/httprl.admin.inc +++ b/httprl.admin.inc @@ -15,6 +15,44 @@ function httprl_admin_settings_form() { '#default_value' => variable_get('httprl_server_addr', FALSE), '#description' => t('If left blank it will use the same server as the request. If set to -1 it will use the host name instead of an IP address. This controls the output of httprl_build_url_self()'), ); + $form['timeous'] = array( + '#type' => 'fieldset', + '#title' => t('Default Timeouts'), + '#collapsible' => TRUE, + '#collapsed' => TRUE, + '#description' => t('Set the default timeouts. These will be overridden if a different timeout is set in the options array.'), + ); + $form['timeous']['httprl_dns_timeout'] = array( + '#type' => 'textfield', + '#title' => t('dns_timeout. Maximum number of seconds a DNS lookup request may take'), + '#default_value' => variable_get('httprl_dns_timeout', HTTPRL_DNS_TIMEOUT), + '#description' => t('Value can be a float. The default is 5 seconds. '), + ); + $form['timeous']['httprl_connect_timeout'] = array( + '#type' => 'textfield', + '#title' => t('connect_timeout. Maximum number of seconds establishing the TCP connection may take'), + '#default_value' => variable_get('httprl_connect_timeout', HTTPRL_CONNECT_TIMEOUT), + '#description' => t('Value can be a float. The default is 5 seconds.'), + ); + $form['timeous']['httprl_ttfb_timeout'] = array( + '#type' => 'textfield', + '#title' => t('ttfb_timeout. Maximum number of seconds a connection may take to download the first byte'), + '#default_value' => variable_get('httprl_ttfb_timeout', HTTPRL_TTFB_TIMEOUT), + '#description' => t('Value can be a float. The default is 20 seconds.'), + ); + $form['timeous']['httprl_timeout'] = array( + '#type' => 'textfield', + '#title' => t('timeout. Maximum number of seconds the request may take'), + '#default_value' => variable_get('httprl_timeout', HTTPRL_TIMEOUT), + '#description' => t('Value can be a float. The default is 30 seconds.'), + ); + $form['timeous']['httprl_global_timeout'] = array( + '#type' => 'textfield', + '#title' => t('global_timeout. Maximum number of seconds httprl_send_request() may take'), + '#default_value' => variable_get('httprl_global_timeout', HTTPRL_GLOBAL_TIMEOUT), + '#description' => t('Value can be a float. The default is 120 seconds.'), + ); + return system_settings_form($form); } @@ -23,13 +61,42 @@ function httprl_admin_settings_form() { * Validate form values. */ function httprl_admin_settings_form_validate($form, &$form_state) { - $advagg_server_addr = $form_state['values']['httprl_server_addr']; + // Skip validation if we are resting to defaults. + if ($form_state['clicked_button']['#post']['op'] == 'Reset to defaults') { + return; + } - // If the IP field is not blank, check that its a valid address. - if ( !empty($advagg_server_addr) - && $advagg_server_addr != -1 - && ip2long($advagg_server_addr) === FALSE + // Get form values. + $values = $form_state['values']; + + // If the IP field is not blank, check that it is a valid address. + if ( !empty($values['httprl_server_addr']) + && $values['httprl_server_addr'] != -1 + && ip2long($values['httprl_server_addr']) === FALSE ) { form_set_error('httprl_server_addr', t('Must be a valid IP address.')); } + + // Make sure the timeouts are positive numbers. + $positive_values = array( + 'httprl_dns_timeout', + 'httprl_connect_timeout', + 'httprl_ttfb_timeout', + 'httprl_timeout', + 'httprl_global_timeout', + ); + foreach ($positive_values as $name) { + if (empty($values[$name])) { + form_set_error($name, t('Must not be empty or zero.')); + continue; + } + if (!is_numeric($values[$name])) { + form_set_error($name, t('Must be numeric.')); + continue; + } + if ($values[$name] <= 0) { + form_set_error($name, t('Must be a postive number.')); + continue; + } + } } diff --git a/httprl.module b/httprl.module index 1fa9ebb..65e7b8e 100644 --- a/httprl.module +++ b/httprl.module @@ -10,6 +10,23 @@ define('HTTPRL_TIMEOUT', 30.0); /** + * Default maximum number of seconds the DNS portion of a request may take. + */ +define('HTTPRL_DNS_TIMEOUT', 5.0); + +/** + * Default maximum number of seconds establishing the TCP connection of a + * request may take. + */ +define('HTTPRL_CONNECT_TIMEOUT', 5.0); + +/** + * Default maximum number of seconds a connection may take to download the first + * byte. + */ +define('HTTPRL_TTFB_TIMEOUT', 20.0); + +/** * Default maximum number of seconds a function call may take. */ define('HTTPRL_GLOBAL_TIMEOUT', 120.0); @@ -289,14 +306,17 @@ function httprl_set_default_options(&$options) { 'method' => 'GET', 'data' => NULL, 'max_redirects' => 3, - 'timeout' => HTTPRL_TIMEOUT, + 'timeout' => httprl_variable_get('httprl_timeout', HTTPRL_TIMEOUT), + 'dns_timeout' => httprl_variable_get('httprl_dns_timeout', HTTPRL_DNS_TIMEOUT), + 'connect_timeout' => httprl_variable_get('httprl_connect_timeout', HTTPRL_CONNECT_TIMEOUT), + 'ttfb_timeout' => httprl_variable_get('httprl_ttfb_timeout', HTTPRL_TTFB_TIMEOUT), 'context' => NULL, 'blocking' => TRUE, 'version' => '1.0', 'referrer' => FALSE, 'domain_connections' => 2, 'global_connections' => 128, - 'global_timeout' => HTTPRL_GLOBAL_TIMEOUT, + 'global_timeout' => httprl_variable_get('httprl_global_timeout', HTTPRL_GLOBAL_TIMEOUT), 'chunk_size_read' => 32768, 'chunk_size_write' => 1024, 'async_connect' => TRUE, @@ -349,7 +369,7 @@ function httprl_setup_proxy(&$uri, &$options, $url) { // Add in username and password to Proxy-Authorization header if needed. if ($proxy_username = httprl_variable_get('proxy_username', '')) { $proxy_password = httprl_variable_get('proxy_password', ''); - $options['headers']['Proxy-Authorization'] = 'Basic ' . base64_encode($proxy_username . (!empty($proxy_password) ? ":" . $proxy_password : '')); + $options['headers']['Proxy-Authorization'] = 'Basic ' . base64_encode($proxy_username . ':' . $proxy_password); } // Some proxies reject requests with any User-Agent headers, while others // require a specific one. @@ -515,7 +535,7 @@ function httprl_handle_data(&$options) { // TODO: mime detection. $mimetype = 'application/octet-stream'; - // Build the datastream for this file. + // Build the data-stream for this file. $data_stream .= '--' . HTTPRL_MULTIPART_BOUNDARY . "\r\n"; $data_stream .= 'Content-Disposition: form-data; name="files[' . $field_name . ']' . $multi_field . '"; filename="' . $filename . "\"\r\n"; $data_stream .= 'Content-Transfer-Encoding: binary' . "\r\n"; @@ -592,7 +612,7 @@ function httprl_multipart_encoder(&$data_stream, $data_array, $prepend = array() function httprl_basic_auth($uri, &$options) { // If the server URL has a user then attempt to use basic authentication. if (isset($uri['user'])) { - $options['headers']['Authorization'] = 'Basic ' . base64_encode($uri['user'] . (!empty($uri['pass']) ? ":" . $uri['pass'] : '')); + $options['headers']['Authorization'] = 'Basic ' . base64_encode($uri['user'] . ':' . (isset($uri['pass']) ? $uri['pass'] : '')); } } @@ -680,6 +700,11 @@ function httprl_stream_connection_error_formatter($errno, $errstr, &$result) { $result->error = $errstr; } } + elseif ($errno == 110) { + // 110 means Connection timed out. This should be HTTPRL_REQUEST_TIMEOUT. + $result->code = HTTPRL_REQUEST_TIMEOUT; + $result->error = !empty($errstr) ? $errstr : $t('Connection timed out. TCP.'); + } else { // When a network error occurs, we use a negative number so it does not // clash with the HTTP status codes. @@ -722,14 +747,21 @@ function httprl_establish_stream_connection(&$result) { } } + // Set the DNS timeout. + $timeout = $result->options['dns_timeout']; + // If not using async_connect then add connect_timeout to timeout. + if (!$result->options['async_connect']) { + $timeout += $result->options['connect_timeout']; + } + // Open the connection. if (empty($result->options['context'])) { - $result->fp = @stream_socket_client($result->socket, $errno, $errstr, $result->options['timeout'], $result->flags); + $result->fp = @stream_socket_client($result->socket, $errno, $errstr, $timeout, $result->flags); } else { // Create a stream with context. Context allows for the verification of // a SSL certificate. - $result->fp = @stream_socket_client($result->socket, $errno, $errstr, $result->options['timeout'], $result->flags, $result->options['context']); + $result->fp = @stream_socket_client($result->socket, $errno, $errstr, $timeout, $result->flags, $result->options['context']); } $count++; } @@ -746,7 +778,7 @@ function httprl_establish_stream_connection(&$result) { $result->fp = FALSE; } - // Report any errors or set the steram to non blocking mode. + // Report any errors or set the stream to non blocking mode. if (!$result->fp) { httprl_stream_connection_error_formatter($errno, $errstr, $result); } @@ -794,7 +826,18 @@ function httprl_establish_stream_connection(&$result) { * may be followed. Defaults to 3. * - timeout: A float representing the maximum number of seconds a connection * may take. The default is 30 seconds. If a timeout occurs, the error code - * is set to the HTTPRL_REQUEST_TIMEOUT constant. + * is set to the HTTPRL_REQUEST_ABORTED constant. + * - dns_timeout: A float representing the maximum number of seconds a DNS + * lookup request may take. The default is 5 seconds. If a timeout occurs, + * the error code is set to the HTTPRL_HOST_NOT_FOUND constant. + * - connect_timeout: A float representing the maximum number of seconds + * establishing the TCP connection may take. The default is 5 seconds. If a + * timeout occurs, the error code is set to the HTTPRL_REQUEST_TIMEOUT + * constant. + * - ttfb_timeout: A float representing the maximum number of seconds a + * connection may take to download the first byte. The default is 20 + * seconds. If a timeout occurs, the error code is set to the + * HTTPRL_REQUEST_ABORTED constant. * - context: A context resource created with stream_context_create(). * - blocking: set to FALSE to make this not care about the returned data. * - version: HTTP Version 1.0 or 1.1. Default is 1.0 for a good reason. @@ -861,7 +904,7 @@ function httprl_request($urls, $options = array()) { foreach ($urls as $url) { $result = new stdClass(); $result->url = $url; - $result->status = 'in progress'; + $result->status = 'Connecting.'; $result->code = 0; $result->chunk_size = 1024; $result->data = ''; @@ -996,7 +1039,6 @@ function httprl_send_request($results = NULL) { $start_time_this_run = $start_time_global = microtime(TRUE); // Run the loop as long as we have a stream to read/write to. - $empty_runs = 0; $stream_select_timeout = 1; $stream_write_count = 0; @@ -1016,13 +1058,19 @@ function httprl_send_request($results = NULL) { $start_time_this_run = $now; $global_time = $global_timeout - ($start_time_this_run - $start_time_global); - $reset_empty_runs = FALSE; // Inspect each stream, checking for timeouts and connection limits. foreach ($responses as $id => &$result) { // See if function timed out. if ($global_time <= 0) { // Function timed out & the request is not done. - if ($result->status == 'in progress') { + if ($result->status == 'Connecting.') { + $result->error = $t('Function timed out. TCP.'); + // If stream is not done writing, then remove one from the write count. + if (isset($result->fp)) { + $stream_write_count--; + } + } + elseif ($result->status == 'Writing To Server.') { $result->error = $t('Function timed out. Write.'); // If stream is not done writing, then remove one from the write count. if (isset($result->fp)) { @@ -1030,7 +1078,7 @@ function httprl_send_request($results = NULL) { } } else { - $result->error = $t('Function timed out.'); + $result->error = $t('Function timed out. Read'); } $result->code = HTTPRL_FUNCTION_TIMEOUT; $result->status = 'Done.'; @@ -1045,45 +1093,62 @@ function httprl_send_request($results = NULL) { $result->running_time += $elapsed_time; // Calculate how much time is left of the original timeout value. $timeout = $result->options['timeout'] - $result->running_time; - // No streams are ready from stream_select, See if end server has - // dropped the connection, or has failed to make the connection. - $socket_name = 'Not empty.'; - if ($result->options['async_connect'] && $empty_runs > 32) { - // If nothing has happened after 32 runs, see if the connection has - // been made. - $socket_name = stream_socket_get_name($result->fp, TRUE); - } // Connection was dropped or connection timed out. - if ($timeout <= 0 || empty($socket_name)) { + if ($timeout <= 0) { $result->error = $t('Connection timed out.'); - if (empty($socket_name)) { - $result->error .= ' ' . $t('If you believe this is a false error, set async_connect to false in the options array that is passed into httprl_request() and try again.'); - } // Stream timed out & the request is not done. - if ($result->status == 'in progress') { - $result->error .= $t(' Write.'); + if ($result->status == 'Writing To Server.') { + $result->error .= ' ' . $t('Write.'); // If stream is not done writing, then remove one from the write count. $stream_write_count--; } else { - $result->error .= $t(' Read.'); + $result->error .= ' ' . $t('Read.'); } $result->code = HTTPRL_REQUEST_TIMEOUT; $result->status = 'Done.'; // Do post processing on the stream. httprl_post_processing($id, $responses, $output, $timeout); - $reset_empty_runs = TRUE; + continue; + } + + // Connection was dropped or connection timed out. + if ($result->status == 'Connecting.' && $result->running_time > $result->options['connect_timeout']) { + $socket_name = stream_socket_get_name($result->fp, TRUE); + if (empty($socket_name) || $result->running_time > ($result->options['connect_timeout'] * 1.5)) { + $result->error = $t('Connection timed out.'); + // Stream timed out & the request is not done. + if ($result->status == 'Connecting.') { + $result->error .= ' ' . $t('TCP Connect Timeout.'); + // If stream is not done writing, then remove one from the write count. + $stream_write_count--; + } + $result->code = HTTPRL_REQUEST_TIMEOUT; + $result->status = 'Done.'; + + // Do post processing on the stream. + httprl_post_processing($id, $responses, $output, $timeout); + continue; + } + } + + if (!isset($responses[$id]->time_to_first_byte) && $result->running_time > $result->options['ttfb_timeout']) { + $result->error = $t('Connection timed out. Time to First Byte Timeout.'); + $result->code = HTTPRL_REQUEST_ABORTED; + $result->status = 'Done.'; + + // Do post processing on the stream. + httprl_post_processing($id, $responses, $output, $timeout); continue; } } // Connection was handled elsewhere. - if (!isset($result->fp) && $result->status != 'in progress') { + if (!isset($result->fp) && $result->status != 'Connecting.') { // Do post processing on the stream. httprl_post_processing($id, $responses, $output); - $reset_empty_runs = TRUE; continue; } @@ -1105,7 +1170,7 @@ function httprl_send_request($results = NULL) { // If the conditions are correct, let the stream be ran in this loop. if ($global_connection_limit >= $global_connection_count && $domain_connection_limit[$host] >= $domain_connection_count[$host]) { // Establish a new connection. - if (!isset($result->fp) && $result->status == 'in progress') { + if (!isset($result->fp) && $result->status == 'Connecting.') { // Establish a connection to the server. httprl_establish_stream_connection($result); @@ -1141,11 +1206,6 @@ function httprl_send_request($results = NULL) { if (empty($this_run)) { continue; } - if ($reset_empty_runs) { - $empty_runs = 0; - $reset_empty_runs = FALSE; - } - // Set the read and write vars to the streams var. @@ -1159,7 +1219,6 @@ function httprl_send_request($results = NULL) { // We have some streams to read/write to. $rw_done = FALSE; if (!empty($n)) { - $empty_runs = 0; // Readable sockets either have data for us, or are failed connection // attempts. @@ -1181,6 +1240,10 @@ function httprl_send_request($results = NULL) { $chunk = fread($r, $responses[$id]->chunk_size); if (httprl_strlen($chunk) > 0) { $rw_done = TRUE; + if (!isset($responses[$id]->time_to_first_byte)) { + // Calculate Time to First Byte. + $responses[$id]->time_to_first_byte = $result->running_time + microtime(TRUE) - $start_time_this_run; + } } $responses[$id]->data .= $chunk; @@ -1207,7 +1270,7 @@ function httprl_send_request($results = NULL) { // If a range header is set, 200 was returned, and method is GET // calculate how many bytes need to be downloaded. - if (!empty($responses[$id]->options['headers']['Range']) && $responses[$id]->code == 200 && $responses[$id]->method == 'GET') { + if (!empty($responses[$id]->options['headers']['Range']) && $responses[$id]->code == 200 && $responses[$id]->options['method'] == 'GET') { $responses[$id]->ranges = httprl_get_ranges($responses[$id]->options['headers']['Range']); $responses[$id]->options['max_data_size'] = httprl_get_last_byte_from_range($responses[$id]->ranges); } @@ -1239,7 +1302,11 @@ function httprl_send_request($results = NULL) { $info = stream_get_meta_data($r); $alive = !$info['eof'] && !feof($r) && !$info['timed_out'] && httprl_strlen($chunk); if (!$alive) { - if ($responses[$id]->status == 'in progress') { + if ($responses[$id]->status == 'Connecting.') { + $responses[$id]->error = $t('Connection refused by destination. TCP.'); + $responses[$id]->code = HTTPRL_CONNECTION_REFUSED; + } + if ($responses[$id]->status == 'Writing To Server.') { $responses[$id]->error = $t('Connection refused by destination. Write.'); $responses[$id]->code = HTTPRL_CONNECTION_REFUSED; } @@ -1259,7 +1326,7 @@ function httprl_send_request($results = NULL) { foreach ($write as $w) { $id = array_search($w, $this_run); // Make sure ID is in the streams & status is for writing. - if ($id === FALSE || empty($responses[$id]->status) || $responses[$id]->status != 'in progress') { + if ($id === FALSE || empty($responses[$id]->status) || ($responses[$id]->status != 'Connecting.' && $responses[$id]->status != 'Writing To Server.')) { continue; } @@ -1314,6 +1381,10 @@ function httprl_send_request($results = NULL) { $rw_done = TRUE; } else { + // Change status to 'Writing To Server.' + if ($responses[$id]->status = 'Connecting.') { + $responses[$id]->status = 'Writing To Server.'; + } // There is more data to write to this socket. Cut what was sent // across the stream and resend whats left next time in the loop. $responses[$id]->request_left = substr($data_to_send, $bytes); @@ -1322,23 +1393,6 @@ function httprl_send_request($results = NULL) { } } } - else { - $empty_runs++; - } - if ($empty_runs > 400) { - // If stream_select hasn't returned a valid read or write stream after - // 10+ seconds, error out. - foreach ($this_run as $id => $fp) { - // stream_select timed out & the request is not done. - $responses[$id]->error = $t('stream_select() timed out.'); - $responses[$id]->code = HTTPRL_STREAM_SELECT_TIMEOUT; - $responses[$id]->status = 'Done.'; - - // Do post processing on the stream. - httprl_post_processing($id, $responses, $output); - continue; - } - } if (!$rw_done) { // Wait 5ms for data buffers. usleep(5000); @@ -1489,6 +1543,8 @@ function httprl_parse_data(&$result) { switch ($code) { case 200: // OK + case 201: // Created + case 202: // Accepted case 206: // Partial Content case 304: // Not modified break; @@ -1564,7 +1620,7 @@ function httprl_get_ranges($input) { // Make sure the input string matches the correct format. $string = preg_match('/^bytes=((\d*-\d*,? ?)+)$/', $input, $matches) ? $matches[1] : FALSE; if (!empty($string)) { - // Handle mutiple ranges + // Handle multiple ranges foreach (explode(',', $string) as $range) { // Get the start and end byte values for this range. $values = explode('-', $range); @@ -1581,7 +1637,7 @@ function httprl_get_ranges($input) { * Given an array of ranges, get the last byte we need to download. * * @param $ranges - * Multi dimentional array + * Multi dimensional array * @return int or NULL * NULL: Get all values; int: last byte to download. */ @@ -2528,7 +2584,7 @@ function httprl_call_exit() { * * @return Bool * TRUE if DRUPAL_BOOTSTRAP_FULL. - * FALse if not DRUPAL_BOOTSTRAP_FULL. + * FALSE if not DRUPAL_BOOTSTRAP_FULL. */ function httprl_drupal_full_bootstrap() { static $full_bootstrap;