diff --git a/README.txt b/README.txt index 529b502..c8f90ba 100644 --- a/README.txt +++ b/README.txt @@ -243,3 +243,35 @@ which is more advanced and faster, by adding the following to settings.php: $conf['memcache_options'] = array( Memcached::OPT_BINARY_PROTOCOL => TRUE, ); + + +## Stampede protection +Memcache now includes stampede protection for expired and invalid cache items. +To enable stampede protection, enable it in settings.php +$conf['memcache_stampede_protection'] = TRUE; + +Stampede protection relies on the locking framework. It is strongly recommended +to use the memcache lock implementation instead of core's SQL implementation. +This is especially true if using the stampede protection since a lock stampede +may be as bad or worse than a cache stampede if using SQL. +$conf['lock_inc'] = './sites/all/modules/memcache/memcache-lock.inc'; + +Behaviour of the stampede protection can be tweaked via the following, see +comments in memcache.inc for more. + +The value passed to lock_acquire. Defaults to '15'. +$conf['memcache_stampede_semaphore'] = 15; + +The value to pass to lock_wait, defaults to 5. +$conf['memcache_stampede_wait_time'] = 5; + +The limit of calls to lock_wait() due to stampede protection during one request. +Defaults to 3. +$conf['memcache_stampede_wait_limit'] = 3; + +When setting these variables, note that: + - there is unlikely to be a good use case for setting wait_time higher + than stampede_semaphore. + - wait_time * wait_limit is designed to default to a number less than + standard web server timeouts (i.e. 15 seconds vs. apache's default of + 30 seconds). diff --git a/dmemcache.inc b/dmemcache.inc index 83d3611..884ca05 100644 --- a/dmemcache.inc +++ b/dmemcache.inc @@ -101,24 +101,7 @@ function dmemcache_get($key, $bin = 'cache') { $success = '0'; if ($mc = dmemcache_object($bin)) { $result = $mc->get($full_key); - if ($result) { - // We check $result->expire to see if the object has expired. If so, we - // try and grab a lock. If we get the lock, we return FALSE instead of - // the cached object which should cause it to be rebuilt. If we do not - // get the lock, we return the cached object. The goal here is to avoid - // cache stampedes. - // By default the cache stampede semaphore is held for 15 seconds. This - // can be adjusted by setting the memcache_stampede_semaphore variable. - // TODO: Can we log when a sempahore expires versus being intentionally - // freed to track when this is happening? - if (isset($result->expire) && $result->expire !== CACHE_PERMANENT && $result->expire <= $_SERVER['REQUEST_TIME'] && dmemcache_add($full_key .'_semaphore', '', variable_get('memcache_stampede_semaphore', 15))) { - $result = FALSE; - } - else { - $success = '1'; - } - } - $statistics[] = $success; + $statistics[] = (bool) $result; $_memcache_statistics[] = $statistics; return $result; diff --git a/memcache.inc b/memcache.inc index 0970ec0..4cf06af 100644 --- a/memcache.inc +++ b/memcache.inc @@ -40,10 +40,7 @@ function cache_get($cid, $table = 'cache') { // Retrieve the item from the cache. $cache = dmemcache_get($cid, $table); - if (!is_object($cache)) { - return FALSE; - } - // Determine when the current table was last flushed. + // Set up common variables. $cache_flush = variable_get("cache_flush_$table", 0); $cache_content_flush = variable_get("cache_content_flush_$table", 0); $cache_tables = isset($_SESSION['cache_flush']) ? $_SESSION['cache_flush'] : NULL; @@ -51,21 +48,89 @@ function cache_get($cid, $table = 'cache') { $wildcard_flushes = variable_get('memcache_wildcard_flushes', array()); $wildcard_invalidate = variable_get('memcache_wildcard_invalidate', MEMCACHE_WILDCARD_INVALIDATE); - if ($cache->created <= $cache_flush) { - return FALSE; - } - - if ($cache->expire != CACHE_PERMANENT && $cache->created + $cache_lifetime <= $cache_content_flush) { - return FALSE; + if ($cache) { + // Items that have expired are invalid. + if (isset($cache->expire) && $cache->expire !== CACHE_PERMANENT && $cache->expire <= $_SERVER['REQUEST_TIME']) { + // If the memcache_stampede_protection variable is set, allow one process + // to rebuild the cache entry while serving expired content to the + // rest. Note that core happily returns expired cache items as valid and + // relies on cron to expire them, but this is mostly reliant on its + // use of CACHE_TEMPORARY which does not map well to memcache. + // @see http://drupal.org/node/534092 + if (variable_get('memcache_stampede_protection', FALSE)) { + // The process that acquires the lock will get a cache miss, all + // others will get a cache hit. + if (lock_acquire("memcache_$cid:$table", variable_get('memcache_stampede_semaphore', 15))) { + $cache = FALSE; + } + } + else { + $cache = FALSE; + } + } + // Items created before the last full wildcard flush against this bin are + // invalid. + elseif ($cache->created <= $cache_flush) { + $cache = FALSE; + } + // Items created before the last content flush on this bin i.e. + // cache_clear_all() are invalid. + elseif ($cache->expire != CACHE_PERMANENT && $cache->created + $cache_lifetime <= $cache_content_flush) { + $cache = FALSE; + } + // Items cached before the cache was last flushed by the current user are + // invalid. + elseif ($cache->expire != CACHE_PERMANENT && is_array($cache_tables) && isset($cache_tables[$table]) && $cache_tables[$table] >= $cache->created) { + // Cache item expired, return FALSE. + $cache = FALSE; + } + // Finally, check for wildcard clears against this cid. + else { + $flushes = isset($cache->flushes) ? (int)$cache->flushes : 0; + $recorded_flushes = memcache_wildcard_flushes($cid, $table); + if ($flushes < $recorded_flushes) { + $cache = FALSE; + } + // If wildcards are cleared by a partial memcache flush or eviction + // then it is possible for $cache->flushes to be greater than the return + // of memcache_wildcard_flushes(). + if ($flushes > $recorded_flushes) { + // Delete the cache item entirely, it will be set again with the correct + // number of flushes. + dmemcache_delete($cid, $table); + $cache = FALSE; + } + } } - // Items cached before the cache was last flushed by the current user are no - // longer valid. - if ($cache->expire != CACHE_PERMANENT && is_array($cache_tables) && isset($cache_tables[$table]) && $cache_tables[$table] >= $cache->created) { - // Cache item expired, return FALSE. - return FALSE; + // On cache misses, attempt to avoid stampedes when the + // memcache_stampede_protection variable is enabled. + if (!$cache) { + if (variable_get('memcache_stampede_protection', FALSE) && !lock_acquire("memcache_$cid:$table", variable_get('memcache_stampede_semaphore', 15))) { + // Prevent any single request from waiting more than three times due to + // stampede protection. By default this is a maximum total wait of 15 + // seconds. This accounts for two possibilities - a cache and lock miss + // more than once for the same item. Or a cache and lock miss for + // different items during the same request. + // @todo: it would be better to base this on time waited rather than + // number of waits, but the lock API does not currently provide this + // information. Currently the limit will kick in for three waits of 25ms + // or three waits of 5000ms. + static $lock_count = 0; + $lock_count++; + if ($lock_count <= variable_get('memcache_stampede_wait_limit', 3)) { + // The memcache_stampede_semaphore variable was used in previous releases + // of memcache, but the max_wait variable was not, so by default divide + // the sempahore value by 3 (5 seconds). + lock_wait("memcache_$cid:$table", variable_get('memcache_stampede_wait_time', 5)); + return cache_get($cid, $table); + } + } } + // Clean up $_SESSION['cache_flush'] variable array if it is older than + // the minimum cache lifetime, since after that the $cache_flush variable + // will take over. if (is_array($cache_tables) && !empty($cache_tables) && $cache_lifetime) { // Expire the $_SESSION['cache_flush'] variable array if it is older than // the minimum cache lifetime, since after that the $cache_flush variable @@ -75,21 +140,6 @@ function cache_get($cid, $table = 'cache') { $cache_tables = NULL; } } - // Check for wildcard flushes matching this cid. - $flushes = isset($cache->flushes) ? (int)$cache->flushes : 0; - $recorded_flushes = memcache_wildcard_flushes($cid, $table); - if ($flushes < $recorded_flushes) { - return FALSE; - } - // If wildcards are cleared by a partial memcache flush or eviction - // then it is possible for $cache->flushes to be greater than the return - // of memcache_wildcard_flushes(). - if ($flushes > $recorded_flushes) { - // Delete the cache item entirely, it will be set again with the correct - // number of flushes. - dmemcache_delete($cid, $table); - return FALSE; - } return $cache; } @@ -157,6 +207,9 @@ function cache_set($cid, $data, $table = 'cache', $expire = CACHE_PERMANENT, $he // Other requests for the expired object while it is still being rebuilt get // the expired object. dmemcache_set($cid, $cache, 0, $table); + if (isset($GLOBALS['locks']["memcache_$cid:$table"])) { + lock_release("memcache_$cid:$table"); + } } /**