diff --git a/README.txt b/README.txt index 30210d3..3f5e897 100644 --- a/README.txt +++ b/README.txt @@ -669,7 +669,11 @@ Other options you could experiment with: tells the TCP stack to send packets immediately and without waiting for a full payload, reducing per-packet network latency (disabling "Nagling"). -It's possible to enable SASL authentication as documented here: +### Authentication + +#### Binary Protocol SASL Authentication + +SASL authentication can be enabled as documented here: http://php.net/manual/en/memcached.setsaslauthdata.php https://code.google.com/p/memcached/wiki/SASLHowto @@ -686,6 +690,31 @@ memcache_sasl_username and memcache_sasl_password in settings.php. For example: $conf['memcache_sasl_username'] = 'yourSASLUsername'; $conf['memcache_sasl_password'] = 'yourSASLPassword'; +#### ASCII Protocol Authentication + +If you do not want to enable the binary protocol, you can instead enable +token authentication with the default ASCII protocol. + +ASCII protocol authentication requires Memcached version 1.5.15 or greater +started with the -Y flag, and the PECL memcached client. It was originally +documented in the memcache 1.5.15 release notes: + https://github.com/memcached/memcached/wiki/ReleaseNotes1515 + +Additional detail can be found in the protocol documentation: + https://github.com/memcached/memcached/blob/master/doc/protocol.txt + +All your memcached servers need to be started with the -Y option to specify +a local path to an authfile which can contain up to 8 "username:pasword" +pairs, any of which can be used for authentication. For example, a simple +authfile may look as follows: + + foo:bar + +You can then configure your website to authenticate with this username and +password as follows: + + $conf['memcache_ascii_auth'] = 'foo bar'; + ## Amazon Elasticache You can use the Drupal Memcache module to talk with Amazon Elasticache, but to diff --git a/dmemcache.inc b/dmemcache.inc index 18f6109..d51dff3 100644 --- a/dmemcache.inc +++ b/dmemcache.inc @@ -11,6 +11,9 @@ */ define('MEMCACHED_E2BIG', 37); +define('MEMCACHED_CLIENT_ERROR', 9); +define('DRUPAL_MEMCACHE_ASCII_AUTH_KEY', 'drupal:memcache:ascii:auth:key'); +define('DRUPAL_MEMCACHE_ASCII_AUTH_LIFETIME_KEY', 'drupal:memcache:ascii:auth:lifetime:key'); global $_dmemcache_stats; $_dmemcache_stats = array('all' => array(), 'ops' => array()); @@ -48,6 +51,10 @@ function dmemcache_set($key, $value, $exp = 0, $bin = 'cache', $mc = NULL) { if ($mc instanceof Memcached) { $rc = $mc->set($full_key, $value, $exp); if (empty($rc)) { + if ($mc->getResultCode() == MEMCACHED_CLIENT_ERROR && _dmemcache_ensure_ascii_auth($full_key, $mc)) { + $rc = $mc->set($full_key, $value, $exp); + } + // If there was a MEMCACHED_E2BIG error, split the value into pieces // and cache them individually. if ($mc->getResultCode() == MEMCACHED_E2BIG) { @@ -237,6 +244,9 @@ function dmemcache_add($key, $value, $exp = 0, $bin = 'cache', $mc = NULL, $flag if ($mc || ($mc = dmemcache_object($bin))) { if ($mc instanceof Memcached) { $rc = $mc->add($full_key, $value, $exp); + if (empty($rc) && $mc->getResultCode() == MEMCACHED_CLIENT_ERROR && _dmemcache_ensure_ascii_auth($full_key, $mc)) { + $rc = $mc->add($full_key, $value, $exp); + } } else { $rc = $mc->add($full_key, $value, $flag, $exp); @@ -274,6 +284,14 @@ function dmemcache_get($key, $bin = 'cache', $mc = NULL) { $php_errormsg = ''; $result = @$mc->get($full_key); + if (empty($result) && _dmemcache_use_ascii_auth()) { + // As sets are costly we first check that the server is not authorized first. + $rc = $mc->getByKey($full_key, DRUPAL_MEMCACHE_ASCII_AUTH_LIFETIME_KEY); + if (empty($rc) && _dmemcache_ensure_ascii_auth($full_key, $mc)) { + $result = @$mc->get($full_key); + } + } + // This is a multi-part value. if (is_object($result) && !empty($result->multi_part_data)) { $result = _dmemcache_get_pieces($result->data, $result->cid, $bin, $mc); @@ -393,6 +411,25 @@ function dmemcache_get_multi($keys, $bin = 'cache', $mc = NULL) { $cas_tokens = NULL; $results = $mc->getMulti($full_keys, $cas_tokens, Memcached::GET_PRESERVE_ORDER); } + + // If $results is FALSE, convert it to an empty array. + if (!$results) { + $results = array(); + } + + // Find out which results have values and which have not. + $filtered_results = array_filter($results); + + if (count($filtered_results) < count($full_keys) && _dmemcache_use_ascii_auth()) { + // Convert empty results to the full keys with value of FALSE. + if (empty($results)) { + $results = array_fill_keys(array_values($full_keys), FALSE); + } + $new_results = _dmemcache_get_multi_2nd_chance_ascii_auth($results, $mc); + foreach ($new_results as $full_key => $value) { + $results[$full_key] = $value; + } + } } elseif ($mc instanceof Memcache) { $track_errors = ini_set('track_errors', '1'); @@ -467,6 +504,9 @@ function dmemcache_delete($key, $bin = 'cache', $mc = NULL) { if ($mc || ($mc = dmemcache_object($bin))) { foreach ($full_keys as $fk) { $rc = $mc->delete($fk, 0); + if (empty($rc) && $mc instanceof Memcached && _dmemcache_ensure_ascii_auth($fk, $mc)) { + $rc = $mc->delete($fk, 0); + } if ($rc) { // If the delete succeeded, we now check to see if this item has @@ -542,6 +582,11 @@ function dmemcache_flush($bin = 'cache', $mc = NULL) { $rc = FALSE; if ($mc || ($mc = dmemcache_object($bin))) { $rc = $mc->flush(); + // If a composite call fails, we need to reset the authentication + // for the whole mc object. + if (empty($rc) && _dmemcache_reset_ascii_auth($mc)) { + $mc->flush(); + } } if ($collect_stats) { @@ -575,6 +620,11 @@ function dmemcache_stats($stats_bin = 'cache', $stats_type = 'default', $aggrega if ($mc = dmemcache_object($bin)) { if ($mc instanceof Memcached) { $stats[$bin] = $mc->getStats(); + // If a composite call fails, we need to reset the authentication + // for the whole mc object. + if (empty($stats[$bin]) && _dmemcache_reset_ascii_auth($mc)) { + $stats[$bin] = $mc->getStats(); + } } // The PHP Memcache extension 3.x version throws an error if the stats // type is NULL or not in {reset, malloc, slabs, cachedump, items, @@ -797,6 +847,7 @@ function dmemcache_instance($bin = 'cache') { */ function dmemcache_connect($memcache, $server, $weight) { static $memcache_persistent = NULL; + static $memcache_ascii_auth = NULL; $extension = dmemcache_extension(); @@ -808,6 +859,9 @@ function dmemcache_connect($memcache, $server, $weight) { if (!isset($memcache_persistent)) { $memcache_persistent = variable_get('memcache_persistent', TRUE); } + if (!isset($memcache_ascii_auth)) { + $memcache_ascii_auth = _dmemcache_use_ascii_auth(); + } $port_error = FALSE; if ($extension == 'Memcache') { @@ -853,6 +907,9 @@ function dmemcache_connect($memcache, $server, $weight) { } if (!$match) { $rc = $memcache->addServer($host, $port); + if ($rc && $memcache_ascii_auth) { + $rc = _dmemcache_ascii_authenticate_server($host, $port, $memcache); + } } else { $rc = TRUE; @@ -1265,3 +1322,221 @@ function _dmemcache_write_debug($action, $bin, $key, $rc) { } } } + +/** + * Ensures memcache connection is authorized for the server + * that is mapped for the given key. + * + * @param string $full_key + * The full key for the item whose server is checked for ascii + * authentication. + * @param object $mc + * The memcache object. + * + * @return bool + * TRUE if the connection is authorized, FALSE otherwise. + */ +function _dmemcache_ensure_ascii_auth($full_key, $mc) { + static $memcache_ascii_auth = NULL; + + if (!isset($memcache_ascii_auth)) { + $memcache_ascii_auth = _dmemcache_use_ascii_auth(); + } + + // Immediately return if there is no memcache authentication. + if (!$memcache_ascii_auth) { + return FALSE; + } + + // Login to the server. + $rc = $mc->setByKey($full_key, DRUPAL_MEMCACHE_ASCII_AUTH_KEY, $memcache_ascii_auth); + if (!$rc) { + $s = $mc->getServerByKey($full_key); + register_shutdown_function('watchdog', 'memcache', 'Memcache ascii authentication failed to login to server: %server', array( + '%server' => $s['host'] . ':' . $s['port'], + ), WATCHDOG_ERROR); + + return FALSE; + } + + // Login was successful, check if our lifetime key exists already. + $rc = $mc->getByKey($full_key, DRUPAL_MEMCACHE_ASCII_AUTH_LIFETIME_KEY); + if ($rc) { + return TRUE; + } + + // Set the lifetime key. + $mc->setByKey($full_key, DRUPAL_MEMCACHE_ASCII_AUTH_LIFETIME_KEY, TRUE); + + // Confirm the lifetime key properly set, setByKey() can return success + // even when it fails. + return $mc->getByKey($full_key, DRUPAL_MEMCACHE_ASCII_AUTH_LIFETIME_KEY); +} + +/** + * Performs ascii authentication to the server given by host and port. + * + * @param string $host + * The hostname of the server to authenticate to. + * @param int $port + * The port of the server to authenticate to. + * @param object $mc + * The memcache object. + * + * @return bool + * TRUE on success, FALSE otherwise. + */ +function _dmemcache_ascii_authenticate_server($host, $port, $mc) { + $rc = FALSE; + + // Find a working key that matches the specified server. + for ($key = 0; $key < 1000; $key++) { + // It's impossible to directly address a server, but we need to + // login to the specified connection before it can be used. + // So we check all keys from 0 to 1000 if it would map to this server. + // Once we have found one, we set the authentication data, because any key + // will work for authentication as the authentication data includes user and + // password. + $s = $mc->getServerByKey($key); + if ($s['host'] == $host && $s['port'] == $port) { + $rc = _dmemcache_ensure_ascii_auth($key, $mc); + break; + } + } + + // This should never happen. + if ($key == 1000) { + register_shutdown_function('watchdog', 'memcache', 'Memcache ascii authentication could not find the server to authenticate to.', array(), WATCHDOG_ERROR); + return FALSE; + } + + return $rc; +} + +/** + * Checks if cache items are really missing or only missing ascii authentication. + * + * This is essentially a cache miss and hence we would be going into a heavy + * code path anyway, therefore it is performance wise okay to check a special + * key that exists on all memcache servers to see if this is: + * + * a) real cache miss (where we need to do all the hard work) + * b) ascii authentication failure (where we just need to login and then can + * hopefully have a cache hit) + * + * The edge case of the special lifetime key not existing is treated as a real + * cache miss. + * + * This is justified by the case that most likely memcache was just restarted + * (else the lifetime key would not be missing) and hence the real cache request + * likely would have been a real cache miss anyway. + * + * @param array $results + * The results array to check for a 2nd chance. + * @param object $mc + * The memcache object. + */ +function _dmemcache_get_multi_2nd_chance_ascii_auth($results, $mc) { + // Cache misses are very costly: Find out, if any servers need authentication. + $servers = array(); + foreach ($results as $full_key => $value) { + if (!empty($value)) { + continue; + } + + $s = $mc->getServerByKey($full_key); + $server_key = $s['host'] . ':' . $s['port']; + $servers[$server_key][$full_key] = $full_key; + } + + if (empty($servers)) { + return array(); + } + + $try_again_full_keys = array(); + foreach ($servers as $server_key => $full_keys) { + // Use the first key of the list as all keys map to this server. + $full_key = current($full_keys); + + // Check if lifetime token exists and hence if we are authorized + // on this server. + $rc = $mc->getByKey($full_key, DRUPAL_MEMCACHE_ASCII_AUTH_LIFETIME_KEY); + if (empty($rc) && _dmemcache_ensure_ascii_auth($full_key, $mc)) { + // If we logged in after first failing a fetch for our lifetime key, + // then these keys need to be retried as the "not found" could have been + // an authentication failure. + $try_again_full_keys += $full_keys; + } + } + + // In case no authentication succeeded, there will be no keys to get. + if (empty($try_again_full_keys)) { + return array(); + } + + $new_results = $mc->getMulti($try_again_full_keys); + if (empty($new_results)) { + return array(); + } + + return $new_results; +} + +/** + * If using ascii authentication, re-authenticates all servers. + * + * @param string $mc + * The mc object to re-authenticate all servers on. + * + * @return bool + * TRUE on success, FALSE on failure. + */ +function _dmemcache_reset_ascii_auth($mc, $return_failed_server = FALSE) { + static $memcache_ascii_auth = NULL; + + if (!isset($memcache_ascii_auth)) { + $memcache_ascii_auth = _dmemcache_use_ascii_auth(); + } + + if (!$memcache_ascii_auth) { + return FALSE; + } + + // Need to check all servers for authentication. + $servers = $mc->getServerList(); + foreach ($servers as $s) { + $rc = _dmemcache_ascii_authenticate_server($s['host'], $s['port'], $mc); + if (!$rc) { + if ($return_failed_server) { + return $s; + } + else { + return FALSE; + } + } + } + + return TRUE; +} + +/** + * Returns whether memcache_ascii_auth is used or not. + * + * @return bool + * TRUE if authentication is used, FALSE otherwise. + */ +function _dmemcache_use_ascii_auth() { + static $memcache_ascii_auth = NULL; + + if (!isset($memcache_ascii_auth)) { + $memcache_ascii_auth = variable_get('memcache_ascii_auth', FALSE); + $extension = dmemcache_extension(); + + if ($memcache_ascii_auth && $extension == 'Memcache') { + register_shutdown_function('watchdog', 'memcache', 'Memcache ascii authentication can only be used with Memcached extension', array(), WATCHDOG_ERROR); + $memcache_ascii_auth = FALSE; + } + } + + return $memcache_ascii_auth; +} diff --git a/memcache-lock.inc b/memcache-lock.inc index 6a26fcf..53003bd 100644 --- a/memcache-lock.inc +++ b/memcache-lock.inc @@ -16,7 +16,14 @@ $mc = dmemcache_object('semaphore'); // dmemcache_object may return FALSE, we don't need these stats but it forces // us to try and connect to memcache. If this fails, we can't store locks in // memcache. -if (!$mc || !$mc->getStats()) { +$connection_okay = $mc && !empty($mc->getStats()); + +// Reset the server list and try the getStats() call again. +if (!$connection_okay && $mc && _dmemcache_reset_ascii_auth($mc)) { + $connection_okay = !empty($mc->getStats()); +} + +if (!$connection_okay) { $lock_file = DRUPAL_ROOT . '/includes/lock.inc'; } require_once $lock_file; diff --git a/memcache.install b/memcache.install index aee97dc..d068bc1 100644 --- a/memcache.install +++ b/memcache.install @@ -7,6 +7,7 @@ define('MEMCACHE_PECL_RECOMMENDED', '3.0.6'); define('MEMCACHED_PECL_RECOMMENDED', '2.0.1'); +define('MEMCACHED_ASCII_AUTH_MINIMUM', '1.5.15'); /** * Implements hook_enable(). @@ -25,7 +26,7 @@ function memcache_enable() { $error = TRUE; } else { - if (!_memcache_version_valid()) { + if (!_memcache_pecl_version_valid()) { $warning = TRUE; } // Make a test connection to all configured memcache servers. @@ -105,7 +106,7 @@ function memcache_requirements($phase) { } $requirements['memcache_extension']['value'] = $version . _memcache_statistics_link(); - if (!_memcache_version_valid()) { + if (!_memcache_pecl_version_valid()) { $warnings[] = $t('PECL !extension version %version is unsupported. Please update to %recommended or newer.', array( '!extension' => $extension, '%version' => $version, @@ -113,6 +114,40 @@ function memcache_requirements($phase) { )); } + // If ASCII protocol authentication is enabled, perform extra tests. + if (_dmemcache_use_ascii_auth()) { + // Verify minimum required memcached version for ASCII protocol authentication is installed. + $version = _memcached_ascii_auth_version_valid(); + if ($version !== TRUE) { + $errors[] = $t('ASCII protocol authentication is enabled but requires memcached v%minimum or greater. One or more memcached instances detected running memcache v%version.' . _memcache_statistics_link(), array( + '%version' => $version, + '%minimum' => MEMCACHED_ASCII_AUTH_MINIMUM, + )); + } + + // Confirm ASCII authentication works on all memcached servers. + $memcache_bins = variable_get('memcache_bins', array('cache' => 'default')); + foreach ($memcache_bins as $bin => $_) { + if ($mc = dmemcache_object($bin)) { + $ascii_auth = _dmemcache_reset_ascii_auth($mc, TRUE); + if ($ascii_auth !== TRUE) { + if ($ascii_auth === FALSE) { + $errors[] = $t('ASCII protocol authentication failed.'); + } + else { + $errors[] = $t('ASCII protocol authentication failed to %host:%port.' . _memcache_statistics_link(), array( + '%host' => $ascii_auth['host'], + '%port' => $ascii_auth['port'], + )); + } + } + } + } + + + } + + // Make a test connection to all configured memcache servers. $memcache_servers = variable_get('memcache_servers', array('127.0.0.1:11211' => 'default')); foreach ($memcache_servers as $server => $bin) { @@ -194,7 +229,7 @@ function _memcache_statistics_link() { /** * Validate whether the current PECL version is supported. */ -function _memcache_version_valid() { +function _memcache_pecl_version_valid() { $extension = dmemcache_extension(); if ($extension == 'Memcache') { return version_compare(phpversion('memcache'), MEMCACHE_PECL_RECOMMENDED, '>='); @@ -204,6 +239,33 @@ function _memcache_version_valid() { } } +/** + * If ASCII protocol authentication is enabled, validate whether the current memcached + * version meets the minimum requirements. + */ +function _memcached_ascii_auth_version_valid() { + if (_dmemcache_use_ascii_auth()) { + $stats = dmemcache_stats(); + foreach ($stats as $bin => $servers) { + if (!empty($servers)) { + foreach ($servers as $server => $value) { + $version = $value['version']; + if (version_compare($version, MEMCACHED_ASCII_AUTH_MINIMUM, '<')) { + // Return detected unsupported version of memcached. + return $version; + } + } + } + else { + return FALSE; + } + } + } + // Fall through if no unsupported version of memcached was detected, or ascii auth is + // not enabled. + return TRUE; +} + /** * Remove the memcache_widlcard_flushes variable since its structure has changed. */