From 1faf57dcdda0e9e9779efb42646d137efda21f13 Mon Sep 17 00:00:00 2001 From: Carlos Rodriguez Date: Fri, 14 Oct 2011 11:21:32 -0700 Subject: [PATCH 1/1] Implement cache tags using counter-based invalidation. --- includes/cache-install.inc | 32 +++++++- includes/cache.inc | 163 +++++++++++++++++++++++++++++++--- includes/database/database.inc | 20 ++++ includes/database/mysql/database.inc | 3 + modules/block/block.install | 5 +- modules/field/field.install | 5 +- modules/filter/filter.install | 5 +- modules/simpletest/tests/cache.test | 31 +++++++ modules/system/system.install | 53 +++-------- 9 files changed, 258 insertions(+), 59 deletions(-) diff --git a/includes/cache-install.inc b/includes/cache-install.inc index 8bcf8b7..a4a37d9 100644 --- a/includes/cache-install.inc +++ b/includes/cache-install.inc @@ -23,7 +23,7 @@ class DrupalFakeCache extends DrupalDatabaseCache implements DrupalCacheInterfac return array(); } - function set($cid, $data, $expire = CACHE_PERMANENT) { + function set($cid, $data, $expire = CACHE_PERMANENT, $tags = array()) { } function deletePrefix($cid) { @@ -36,6 +36,36 @@ class DrupalFakeCache extends DrupalDatabaseCache implements DrupalCacheInterfac } } + function delete($cid) { + try { + if (class_exists('Database')) { + parent::delete($cid); + } + } + catch (Exception $e) { + } + } + + function deleteMultiple(array $cids) { + try { + if (class_exists('Database')) { + parent::deleteMultiple($cids); + } + } + catch (Exception $e) { + } + } + + function invalidate($tags) { + try { + if (class_exists('Database')) { + parent::invalidate($tags); + } + } + catch (Exception $e) { + } + } + function clear($cid = NULL, $wildcard = FALSE) { // If there is a database cache, attempt to clear it whenever possible. The // reason for doing this is that the database cache can accumulate data diff --git a/includes/cache.inc b/includes/cache.inc index fcf3e5e..04f706a 100644 --- a/includes/cache.inc +++ b/includes/cache.inc @@ -276,8 +276,14 @@ interface DrupalCacheInterface { * general cache wipe. * - A Unix timestamp: Indicates that the item should be kept at least until * the given time, after which it behaves like CACHE_TEMPORARY. + * @param $tags + * An array of tags to be stored with the cache item. These should normally + * identify objects used to build the cache item, which should trigger + * cache invalidation when updated. For example if a cached item represents + * a node, both the node ID and the author's user ID might be passed in as + * tags. For example array('node' => array(123), 'user' => array(92)). */ - function set($cid, $data, $expire = CACHE_PERMANENT); + function set($cid, $data, $expire = CACHE_PERMANENT, $tags = array()); /** * Delete an item from the cache. @@ -304,6 +310,15 @@ interface DrupalCacheInterface { function deletePrefix($prefix); /** + * Invalidate cached items matching cache tags. + * + * @param $tags + * An array of tags grouped by namespace. The array follows the same format + * as that used to in DrupalCacheInterface::set(). + */ + function invalidate($tags); + + /** * Flush all cache items in a bin. */ function flush(); @@ -370,7 +385,7 @@ class DrupalNullCache implements DrupalCacheInterface { return array(); } - function set($cid, $data, $expire = CACHE_PERMANENT) {} + function set($cid, $data, $expire = CACHE_PERMANENT, $tags = array()) {} function delete($cid) {} @@ -378,6 +393,8 @@ class DrupalNullCache implements DrupalCacheInterface { function deletePrefix($prefix) {} + function invalidate($tags) {} + function flush() {} function expire() {} @@ -399,6 +416,7 @@ class DrupalNullCache implements DrupalCacheInterface { */ class DrupalDatabaseCache implements DrupalCacheInterface { protected $bin; + protected $tags; function __construct($bin) { // All cache tables should be prefixed with 'cache_', apart from the @@ -427,7 +445,7 @@ class DrupalDatabaseCache implements DrupalCacheInterface { // is used here only due to the performance overhead we would incur // otherwise. When serving an uncached page, the overhead of using // db_select() is a much smaller proportion of the request. - $result = db_query('SELECT cid, data, created, expire, serialized FROM {' . db_escape_table($this->bin) . '} WHERE cid IN (:cids)', array(':cids' => $cids)); + $result = db_query('SELECT cid, data, created, expire, serialized, tags, checksum FROM {' . db_escape_table($this->bin) . '} WHERE cid IN (:cids)', array(':cids' => $cids)); $cache = array(); foreach ($result as $item) { $item = $this->prepareItem($item); @@ -474,6 +492,14 @@ class DrupalDatabaseCache implements DrupalCacheInterface { return FALSE; } + // The cache data is invalid if any of its tags have been cleared since. + if ($cache->tags) { + $cache->tags = explode(' ', $cache->tags); + if ($cache->checksum < $this->checksum($cache->tags)) { + return FALSE; + } + } + if ($cache->serialized) { $cache->data = unserialize($cache->data); } @@ -481,11 +507,14 @@ class DrupalDatabaseCache implements DrupalCacheInterface { return $cache; } - function set($cid, $data, $expire = CACHE_PERMANENT) { + function set($cid, $data, $expire = CACHE_PERMANENT, $tags = array()) { + $tags = $this->flattenTags($tags); $fields = array( 'serialized' => 0, 'created' => REQUEST_TIME, 'expire' => $expire, + 'tags' => implode(' ', $tags), + 'checksum' => $this->checksum($tags), ); if (!is_string($data)) { $fields['data'] = serialize($data); @@ -501,6 +530,12 @@ class DrupalDatabaseCache implements DrupalCacheInterface { ->key(array('cid' => $cid)) ->fields($fields) ->execute(); + // Set cache data should be immediately gettable by the current process, + // even if a tag clear was done in the same second (if the clear came before + // the set/get). This bypasses the tag db check for these tags. + foreach ($tags as $tag) { + $this->tags[$tag] = -1; + } } catch (Exception $e) { // The database may not be available, so we'll ignore cache_set requests. @@ -508,19 +543,13 @@ class DrupalDatabaseCache implements DrupalCacheInterface { } function delete($cid) { - db_delete($this->bin) - ->condition('cid', $cid) - ->execute(); + $this->deleteMultiple(array($cid)); } function deleteMultiple(Array $cids) { - // Delete in chunks when a large array is passed. - do { - db_delete($this->bin) - ->condition('cid', array_splice($cids, 0, 1000), 'IN') - ->execute(); - } - while (count($cids)); + db_delete($this->bin) + ->condition('cid', $cids, 'IN') + ->execute(); } function deletePrefix($prefix) { @@ -529,6 +558,49 @@ class DrupalDatabaseCache implements DrupalCacheInterface { ->execute(); } + function flattenTags($tags) { + $flat_tags = array(); + foreach ($tags as $namespace => $values) { + foreach ($values as $value) { + $flat_tags[] = "$namespace:$value"; + } + } + return $flat_tags; + } + + function invalidate($tags) { + foreach ($this->flattenTags($tags) as $tag) { + unset($this->tags[$tag]); + db_merge('cache_tags') + ->key(array('tag' => $tag)) + ->fields(array('invalidations' => 1)) + ->expression('invalidations', 'invalidations + 1') + ->execute(); + } + } + + function checksum($flat_tags) { + $checksum = 0; + $query_tags = array(); + + foreach ($flat_tags as $tag) { + if (isset($this->tags[$tag])) { + $checksum += $this->tags[$tag]; + } + else { + $query_tags[] = $tag; + } + } + if ($query_tags) { + if ($db_tags = db_query('SELECT tag, invalidations FROM {cache_tags} WHERE tag IN (:tags)', array(':tags' => $query_tags))->fetchAllKeyed()) { + $this->tags = array_merge($this->tags, $db_tags); + $checksum += array_sum($db_tags); + } + } + + return $checksum; + } + function flush() { db_truncate($this->bin)->execute(); } @@ -615,4 +687,67 @@ class DrupalDatabaseCache implements DrupalCacheInterface { ->fetchField(); return empty($result); } + + /** + * Return the schema definition for this cache bin. + * + * @return + * An array of schema definitions in the format required by hook_schema(). + */ + function schema() { + $schema[$this->bin] = array( + 'description' => 'Cache bin for use with DrupalDatabaseCache.', + 'fields' => array( + 'cid' => array( + 'description' => 'Primary Key: Unique cache ID.', + 'type' => 'varchar', + 'length' => 255, + 'not null' => TRUE, + 'default' => '', + ), + 'data' => array( + 'description' => 'A collection of data to cache.', + 'type' => 'blob', + 'not null' => FALSE, + 'size' => 'big', + ), + 'expire' => array( + 'description' => 'A Unix timestamp indicating when the cache entry should expire, or 0 for never.', + 'type' => 'int', + 'not null' => TRUE, + 'default' => 0, + ), + 'created' => array( + 'description' => 'A Unix timestamp indicating when the cache entry was created.', + 'type' => 'int', + 'not null' => TRUE, + 'default' => 0, + ), + 'serialized' => array( + 'description' => 'A flag to indicate whether content is serialized (1) or not (0).', + 'type' => 'int', + 'size' => 'small', + 'not null' => TRUE, + 'default' => 0, + ), + 'tags' => array( + 'description' => 'Space-separated list of cache tags for this entry.', + 'type' => 'text', + 'size' => 'big', + 'not null' => FALSE, + ), + 'checksum' => array( + 'description' => 'A Unix timestamp indicating when the cache entry was created.', + 'type' => 'int', + 'not null' => TRUE, + 'default' => 0, + ), + ), + 'indexes' => array( + 'expire' => array('expire'), + ), + 'primary key' => array('cid'), + ); + return $schema; + } } diff --git a/includes/database/database.inc b/includes/database/database.inc index bdd65cc..bc77add 100644 --- a/includes/database/database.inc +++ b/includes/database/database.inc @@ -252,6 +252,15 @@ abstract class DatabaseConnection extends PDO { protected $transactionalDDLSupport = FALSE; /** + * Whether this database connection supports JOIN syntax for DML statements. + * + * Set to FALSE by default because few databases support this feature. + * + * @var bool + */ + protected $joinDMLSupport = FALSE; + + /** * An index used to generate unique temporary table names. * * @var integer @@ -1230,6 +1239,17 @@ abstract class DatabaseConnection extends PDO { } /** + * Determines if this driver supports JOIN syntax for DML statements. + * + * @return + * TRUE if this connection supports JOIN syntax for DML statments, FALSE + * otherwise. + */ + public function supportsJoinDML() { + return $this->joinDMLSupport; + } + + /** * Returns the name of the PDO driver for this connection. */ abstract public function databaseType(); diff --git a/includes/database/mysql/database.inc b/includes/database/mysql/database.inc index 7d5d859..0d77783 100644 --- a/includes/database/mysql/database.inc +++ b/includes/database/mysql/database.inc @@ -26,6 +26,9 @@ class DatabaseConnection_mysql extends DatabaseConnection { // MySQL never supports transactional DDL. $this->transactionalDDLSupport = FALSE; + // MySQL supports JOIN on DML statements. + $this->joinDMLSupport = TRUE; + $this->connectionOptions = $connection_options; // The DSN should use either a socket or a host/port. diff --git a/modules/block/block.install b/modules/block/block.install index dea128d..71e1e6b 100644 --- a/modules/block/block.install +++ b/modules/block/block.install @@ -166,8 +166,9 @@ function block_schema() { 'primary key' => array('bid'), ); - $schema['cache_block'] = drupal_get_schema_unprocessed('system', 'cache'); - $schema['cache_block']['description'] = 'Cache table for the Block module to store already built blocks, identified by module, delta, and various contexts which may change the block, such as theme, locale, and caching mode defined for the block.'; + // Create the cache_block bin on behalf of DrupalDatabaseCache(). + $cache = new DrupalDatabaseCache('block'); + $schema += $cache->schema(); return $schema; } diff --git a/modules/field/field.install b/modules/field/field.install index 16b09e1..2075b93 100644 --- a/modules/field/field.install +++ b/modules/field/field.install @@ -162,7 +162,10 @@ function field_schema() { 'deleted' => array('deleted'), ), ); - $schema['cache_field'] = drupal_get_schema_unprocessed('system', 'cache'); + + // Create the cache_field bin on behalf of DrupalDatabaseCache(). + $cache = new DrupalDatabaseCache('field'); + $schema += $cache->schema(); return $schema; } diff --git a/modules/filter/filter.install b/modules/filter/filter.install index da9ecb8..eda1cba 100644 --- a/modules/filter/filter.install +++ b/modules/filter/filter.install @@ -105,8 +105,9 @@ function filter_schema() { ), ); - $schema['cache_filter'] = drupal_get_schema_unprocessed('system', 'cache'); - $schema['cache_filter']['description'] = 'Cache table for the Filter module to store already filtered pieces of text, identified by text format and hash of the text.'; + // Create the cache_filter bin on behalf of DrupalDatabaseCache(). + $cache = new DrupalDatabaseCache('filter'); + $schema += $cache->schema(); return $schema; } diff --git a/modules/simpletest/tests/cache.test b/modules/simpletest/tests/cache.test index 664247b..673adbc 100644 --- a/modules/simpletest/tests/cache.test +++ b/modules/simpletest/tests/cache.test @@ -277,6 +277,37 @@ class CacheClearCase extends CacheTestCase { } /** + * Test clearing using cache tags. + */ + function testClearTags() { + $cache = cache($this->default_bin); + $cache->set('test_cid_clear1', $this->default_value, CACHE_PERMANENT, array('test_tag' => array(1))); + $cache->set('test_cid_clear2', $this->default_value, CACHE_PERMANENT, array('test_tag' => array(1))); + $this->assertTrue($this->checkCacheExists('test_cid_clear1', $this->default_value) + && $this->checkCacheExists('test_cid_clear2', $this->default_value), + t('Two cache items were created.')); + $cache->invalidate(array('test_tag' => array(1))); + $this->assertFalse($this->checkCacheExists('test_cid_clear1', $this->default_value) + || $this->checkCacheExists('test_cid_clear2', $this->default_value), + t('Two caches removed after clearing a cache tag.')); + + $cache->set('test_cid_clear1', $this->default_value, CACHE_PERMANENT, array('test_tag' => array(1))); + $cache->set('test_cid_clear2', $this->default_value, CACHE_PERMANENT, array('test_tag' => array(2))); + $cache->set('test_cid_clear3', $this->default_value, CACHE_PERMANENT, array('test_tag_foo' => array(3))); + $this->assertTrue($this->checkCacheExists('test_cid_clear1', $this->default_value) + && $this->checkCacheExists('test_cid_clear2', $this->default_value) + && $this->checkCacheExists('test_cid_clear3', $this->default_value), + t('Three cached items were created.')); + $cache->invalidate(array('test_tag_foo' => array(3))); + $this->assertTrue($this->checkCacheExists('test_cid_clear1', $this->default_value) + && $this->checkCacheExists('test_cid_clear2', $this->default_value), + t('Cached items not matching the tag were not cleared.')); + + $this->assertFalse($this->checkCacheExists('test_cid_clear3', $this->default_value), + t('Cached item matching the tag was removed.')); + } + + /** * Test clearing using an array. */ function testClearArray() { diff --git a/modules/system/system.install b/modules/system/system.install index 24933e2..05f300d 100644 --- a/modules/system/system.install +++ b/modules/system/system.install @@ -638,57 +638,32 @@ function system_schema() { 'primary key' => array('iid'), ); - $schema['cache'] = array( - 'description' => 'Generic cache table for caching things not separated out into their own tables. Contributed modules may also use this to store cached items.', + // Create cache bins on behalf of DrupalDatabaseCache. + require_once DRUPAL_ROOT . '/includes/cache.inc'; + $bins = array('bootstrap', 'cache', 'form', 'menu', 'page', 'path'); + foreach ($bins as $bin) { + $cache = new DrupalDatabaseCache($bin); + $schema += $cache->schema(); + } + $schema['cache_tags'] = array( + 'description' => 'Cache table for tracking cache tags related to the cache bin.', 'fields' => array( - 'cid' => array( - 'description' => 'Primary Key: Unique cache ID.', + 'tag' => array( + 'description' => 'Namespace-prefixed tag string.', 'type' => 'varchar', 'length' => 255, 'not null' => TRUE, 'default' => '', ), - 'data' => array( - 'description' => 'A collection of data to cache.', - 'type' => 'blob', - 'not null' => FALSE, - 'size' => 'big', - ), - 'expire' => array( - 'description' => 'A Unix timestamp indicating when the cache entry should expire, or 0 for never.', - 'type' => 'int', - 'not null' => TRUE, - 'default' => 0, - ), - 'created' => array( - 'description' => 'A Unix timestamp indicating when the cache entry was created.', - 'type' => 'int', - 'not null' => TRUE, - 'default' => 0, - ), - 'serialized' => array( - 'description' => 'A flag to indicate whether content is serialized (1) or not (0).', + 'invalidations' => array( + 'description' => 'Number incremented when the tag is invalidated.', 'type' => 'int', - 'size' => 'small', 'not null' => TRUE, 'default' => 0, ), ), - 'indexes' => array( - 'expire' => array('expire'), - ), - 'primary key' => array('cid'), + 'primary key' => array('tag'), ); - $schema['cache_bootstrap'] = $schema['cache']; - $schema['cache_bootstrap']['description'] = 'Cache table for data required to bootstrap Drupal, may be routed to a shared memory cache.'; - $schema['cache_form'] = $schema['cache']; - $schema['cache_form']['description'] = 'Cache table for the form system to store recently built forms and their storage data, to be used in subsequent page requests.'; - $schema['cache_page'] = $schema['cache']; - $schema['cache_page']['description'] = 'Cache table used to store compressed pages for anonymous users, if page caching is enabled.'; - $schema['cache_menu'] = $schema['cache']; - $schema['cache_menu']['description'] = 'Cache table for the menu system to store router information as well as generated link trees for various menu/page/user combinations.'; - $schema['cache_path'] = $schema['cache']; - $schema['cache_path']['description'] = 'Cache table for path alias lookup.'; $schema['date_format_type'] = array( 'description' => 'Stores configured date format types.', -- 1.7.4.1