diff --git a/feeds.info b/feeds.info index 4991435..d9ca981 100644 --- a/feeds.info +++ b/feeds.info @@ -14,6 +14,8 @@ test_dependencies[] = rules:rules test_dependencies[] = variable:variable files[] = includes/FeedsConfigurable.inc +files[] = includes/FeedsHTTPCache.inc +files[] = includes/FeedsHTTPCacheItem.inc files[] = includes/FeedsImporter.inc files[] = includes/FeedsSource.inc files[] = libraries/ParserCSV.inc @@ -73,6 +75,7 @@ files[] = tests/feeds_mapper_link.test files[] = tests/feeds_mapper_summary.test files[] = tests/feeds_mapper_taxonomy.test files[] = tests/feeds_tokens.test +files[] = tests/FeedsHTTPCacheTest.test files[] = tests/http_request.test files[] = tests/parser_csv.test diff --git a/includes/FeedsHTTPCache.inc b/includes/FeedsHTTPCache.inc new file mode 100644 index 0000000..f2b5fe5 --- /dev/null +++ b/includes/FeedsHTTPCache.inc @@ -0,0 +1,211 @@ +getCacheDir() . '/' . $cid; + } + + /** + * Deletes a file from the cache directory. + * + * @param string $cid + * The file to delete. + */ + protected function deleteFile($cid) { + $filename = $this->constructFilePath($cid); + if (is_file($filename)) { + drupal_unlink($filename); + } + } + + /** + * Deletes multiple files from the cache directory. + * + * @param array $cids + * The files to delete. + */ + protected function deleteMultipleFiles(array $cids) { + foreach ($cids as $cid) { + $this->deleteFile($cid); + } + } + + /** + * Deletes all files from the cache directory. + */ + protected function deleteAllFiles() { + $file_cache_dir = $this->getCacheDir(); + if (drupal_realpath($file_cache_dir) && file_exists($file_cache_dir)) { + @file_unmanaged_delete_recursive($file_cache_dir); + } + } + + /** + * Deletes files from the cache directory starting with a certain string. + * + * @param string $string + * The string with which the file name should start. + */ + protected function deleteFilesStartingWith($string) { + $mask = '/^' . preg_quote($string) . '/'; + $files_to_delete = file_scan_directory($this->getCacheDir(), $mask); + foreach ($files_to_delete as $file) { + file_unmanaged_delete($file->uri); + } + } + + /** + * {@inheritdoc} + * + * Converts data to a FeedsHTTPCacheItem object, if needed. + */ + protected function prepareItem($cache) { + $cache = parent::prepareItem($cache); + + if (isset($cache->data) && is_object($cache->data) && !($cache->data instanceof FeedsHTTPCacheItem)) { + // Convert to a FeedsHTTPCacheItem object. + $cache->data = new FeedsHTTPCacheItem($cache->data); + } + + return $cache; + } + + /** + * {@inheritdoc} + */ + public function clear($cid = NULL, $wildcard = FALSE) { + if (empty($cid)) { + // @todo add test. + + // Clean up expired cached files. + if (variable_get('cache_lifetime', 0)) { + $cache_flush = variable_get('cache_flush_' . $this->bin, 0); + if ($cache_flush && REQUEST_TIME > ($cache_flush + variable_get('cache_lifetime', 0))) { + // Clear the cache for everyone, cache_lifetime seconds have passed + // since the first request to clear the cache. + // @todo convert db result to a simple array. + $cids = db_select($this->bin) + ->fields($this->bin, array('cid')) + ->condition('expire', CACHE_PERMANENT, '<>') + ->condition('expire', REQUEST_TIME, '<') + ->execute() + ->fetchAllAssoc(); + + $this->deleteMultipleFiles($cids); + } + } + else { + // No minimum cache lifetime, flush all temporary cache entries now. + // @todo convert db result to a simple array. + $cids = db_select($this->bin) + ->fields($this->bin, array('cid')) + ->condition('expire', CACHE_PERMANENT, '<>') + ->condition('expire', REQUEST_TIME, '<') + ->execute() + ->fetchAllAssoc(); + + $this->deleteMultipleFiles($cids); + } + } + else { + if ($wildcard) { + if ($cid == '*') { + $this->deleteAllFiles(); + } + else { + $this->deleteFilesStartingWith($cid); + } + } + elseif (is_array($cid)) { + $this->deleteMultipleFiles($cid); + } + else { + $this->deleteFile($cid); + } + } + + parent::clear($cid, $wildcard); + } + + /** + * {@inheritdoc} + */ + public function set($cid, $data, $expire = CACHE_PERMANENT) { + // Create a cache item to only cache what is needed. + $item = new FeedsHTTPCacheItem($data); + + // If the object contains the raw data, save it to a file. + if (isset($data->data)) { + $file_cache_dir = $this->getCacheDir(); + file_prepare_directory($file_cache_dir, FILE_CREATE_DIRECTORY); + $filename = $this->constructFilePath($cid); + file_put_contents($filename, $data->data); + + $item->setFilePath($filename); + } + + parent::set($cid, $item, $expire); + } + + /** + * {@inheritdoc} + */ + public function isEmpty() { + // Check database first. + if (!parent::isEmpty()) { + return FALSE; + } + + // Check if the cache directory is empty. + $file_cache_dir = $this->getCacheDir(); + + if (is_dir($file_cache_dir)) { + $dir = dir($file_cache_dir); + while (($entry = $dir->read()) !== FALSE) { + if ($entry == '.' || $entry == '..') { + continue; + } + // At least one file is found, so the cache directory is not empty. + $dir->close(); + return FALSE; + } + // No files found. The cache directory is empty. + $dir->close(); + return TRUE; + } + + // No cache dir found. + return TRUE; + } +} diff --git a/includes/FeedsHTTPCacheItem.inc b/includes/FeedsHTTPCacheItem.inc new file mode 100644 index 0000000..e460a1a --- /dev/null +++ b/includes/FeedsHTTPCacheItem.inc @@ -0,0 +1,68 @@ +headers) && is_array($result->headers)) { + $this->headers = $result->headers; + } + if (isset($result->file_path)) { + $this->setFilePath($result->file_path); + } + } + + /** + * Sets file path. + * + * @param string $file_path + * The file path to set. + */ + public function setFilePath($file_path) { + $this->file_path = $file_path; + } + + /** + * Magic isset. + */ + public function __isset($member) { + return isset($this->$member); + } + + /** + * Magic getter. + */ + public function __get($member) { + return $this->$member; + } + + /** + * Gets the cached file from the file system. + */ + public function getFileContents() { + if (file_exists($this->file_path)) { + return file_get_contents($this->file_path); + } + } +} diff --git a/tests/FeedsHTTPCacheTest.test b/tests/FeedsHTTPCacheTest.test new file mode 100644 index 0000000..018c4ad --- /dev/null +++ b/tests/FeedsHTTPCacheTest.test @@ -0,0 +1,412 @@ + 'FeedsHTTPCache class test', + 'description' => 'Covers class FeedsHTTPCache.', + 'group' => 'Feeds', + ); + } + + /** + * {@inheritdoc} + */ + public function setUp() { + parent::setUp(); + + // Create an instance of FeedsHTTPCache to test with. + $this->cache = new FeedsHTTPCache('cache_feeds_http'); + } + + /** + * Creates a dummy HTTP response. + * + * @param bool $with_data + * (optional) If data should be included. + * Defaults to TRUE. + * + * @return object + * The created response. + */ + protected function createHttpResponse($with_data = TRUE) { + $response = new stdClass(); + $response->code = 200; + $response->headers = array( + 'content-type' => 'text/plain; charset=utf-8', + 'x-host' => 'http://www.example.com', + ); + if ($with_data) { + $response->data = static::randomString(255); + } + + return $response; + } + + /** + * Creates a cache record, bypassing the API. + * + * @param string $cid + * (optional) The cache item ID. + * @param object $response + * (optional) The response to save. + * + * @return string + * The cache item ID. + */ + protected function createCacheRecord($cid = NULL, $response = NULL) { + if (empty($cid)) { + $cid = static::randomName(); + } + if (empty($response)) { + $response = $this->createHttpResponse(FALSE); + $response->file_path = static::FEEDS_CACHE_DIR . '/' . $cid; + } + + $record = new stdClass(); + $record->cid = $cid; + $record->data = serialize($response); + $record->expire = CACHE_PERMANENT; + $record->created = REQUEST_TIME; + $record->serialized = TRUE; + + drupal_write_record('cache_feeds_http', $record); + + return $cid; + } + + /** + * Creates a cache record using the API. + * + * @param string $cid + * (optional) The cache item ID. + * @param object $response + * (optional) The response to save. + * + * @return string + * The cache item ID. + */ + protected function createCacheRecordUsingApi($cid = NULL, $response = NULL) { + if (empty($cid)) { + $cid = static::randomName(); + } + if (empty($response)) { + $response = $this->createHttpResponse(); + } + $this->cache->set($cid, $response); + + return $cid; + } + + /** + * Asserts that an item with a certain ID exists in the feeds cache table. + * + * @param string $cid + * The cache item ID. + */ + protected function assertCacheItemExists($cid) { + $message = format_string('Cache item @cid exists.', array( + '@cid' => $cid, + )); + + $count = db_select('cache_feeds_http') + ->fields('cache_feeds_http', array('cid')) + ->condition('cid', $cid) + ->countQuery() + ->execute() + ->fetchField(); + $this->assertEqual(1, $count, $message); + } + + /** + * Asserts that an item with a certain ID does not exist in the feeds cache + * table. + * + * @param string $cid + * The cache item ID. + */ + protected function assertNoCacheItemExists($cid) { + $message = format_string('Cache item @cid does not exist.', array( + '@cid' => $cid, + )); + + $count = db_select('cache_feeds_http') + ->fields('cache_feeds_http', array('cid')) + ->condition('cid', $cid) + ->countQuery() + ->execute() + ->fetchField(); + $this->assertEqual(0, $count, $message); + } + + /** + * Asserts that a cache file with a certain ID exists on the file system. + * + * @param string $cid + * The cache item ID. + */ + protected function assertCacheFileExists($cid) { + $message = format_string('Cache file @cid exists.', array( + '@cid' => $cid, + )); + + $this->assertTrue(file_exists(static::FEEDS_CACHE_DIR . '/' . $cid), $message); + } + + /** + * Asserts that a cache file with a certain ID does not exist on the file + * system. + * + * @param string $cid + * The cache item ID. + */ + protected function assertNoCacheFileExists($cid) { + $message = format_string('Cache file @cid does not exist.', array( + '@cid' => $cid, + )); + + $this->assertFalse(file_exists(static::FEEDS_CACHE_DIR . '/' . $cid), $message); + } + + /** + * @covers FeedsHTTPCache::get(). + */ + public function testGet() { + $cid = static::randomName(); + $file_data = static::randomName(); + + // Save a file to the cache dir. + $dir = static::FEEDS_CACHE_DIR; + file_prepare_directory($dir, FILE_CREATE_DIRECTORY); + file_put_contents(static::FEEDS_CACHE_DIR . '/' . $cid, $file_data); + + // Manually create a record in cache_feeds_http table. + $this->createCacheRecord($cid); + + // Assert that the item can be get from cache. + $item = $this->cache->get($cid); + $this->assertEqual($file_data, $item->data->getFileContents()); + } + + /** + * @covers FeedsHTTPCache::getMultiple(). + */ + public function testGetMultiple() { + $cid1 = static::randomName(); + $cid2 = static::randomName(); + $cid3 = static::randomName(); + + $file1 = static::randomName(); + $file2 = static::randomName(); + $file3 = static::randomName(); + + // Save a few files to the cache dir. + $dir = static::FEEDS_CACHE_DIR; + file_prepare_directory($dir, FILE_CREATE_DIRECTORY); + file_put_contents(static::FEEDS_CACHE_DIR . '/' . $cid1, $file1); + file_put_contents(static::FEEDS_CACHE_DIR . '/' . $cid2, $file2); + file_put_contents(static::FEEDS_CACHE_DIR . '/' . $cid3, $file3); + + // Create a few records in cache_feeds_http table. + $this->createCacheRecord($cid1); + $this->createCacheRecord($cid2); + $this->createCacheRecord($cid3); + + $cids = array( + $cid2, + $cid3, + ); + + // Assert that the expected items are get from the cache. + $items = $this->cache->getMultiple($cids); + $this->assertFalse(isset($items[$cid1])); + $this->assertTrue(isset($items[$cid2])); + $this->assertTrue(isset($items[$cid3])); + $this->assertEqual($file2, $items[$cid2]->data->getFileContents()); + $this->assertEqual($file3, $items[$cid3]->data->getFileContents()); + } + + /** + * @covers FeedsHTTPCache::set(). + */ + public function testSet() { + $response = $this->createHttpResponse(); + $cid = $this->createCacheRecordUsingApi(NULL, $response); + + // Assert that a record was created with the expected data. + $record = db_select('cache_feeds_http') + ->fields('cache_feeds_http', array()) + ->condition('cid', $cid) + ->execute() + ->fetch(); + + // Check cache record. + $this->assertEqual($cid, $record->cid); + $this->assertEqual(CACHE_PERMANENT, $record->expire); + + // Check that the raw data wasn't saved in the cache record. + $this->assertFalse(strpos($record->data, $response->data), 'The raw data was not saved in the database.'); + + // Check properties. + $saved_response = unserialize($record->data); + $this->assertTrue($saved_response instanceof FeedsHTTPCacheItem, 'Cached data is instance of class FeedsHTTPCacheItem.'); + $this->assertEqual($response->headers, $saved_response->headers); + $this->assertEqual(static::FEEDS_CACHE_DIR . '/' . $cid, $saved_response->file_path); + + // Assert that a file was created on the file system. + $this->assertTrue(file_exists(static::FEEDS_CACHE_DIR . '/' . $cid)); + } + + /** + * @covers FeedsHTTPCache::clear(). + */ + public function testClear() { + // Create a few cache entries. + $cid1 = $this->createCacheRecordUsingApi(); + $cid2 = $this->createCacheRecordUsingApi(); + + // Assert that items were created in the database. + $this->assertCacheItemExists($cid1); + $this->assertCacheItemExists($cid2); + + // Assert that files exist. + $this->assertCacheFileExists($cid1); + $this->assertCacheFileExists($cid2); + + // Now clear first item. + $this->cache->clear($cid1); + + // Assert that item 1 is removed from the database, but item 2 still exists. + $this->assertNoCacheItemExists($cid1); + $this->assertCacheItemExists($cid2); + + // Assert that file 1 is gone as well, but file 2 still exists. + $this->assertNoCacheFileExists($cid1); + $this->assertCacheFileExists($cid2); + } + + /** + * @covers FeedsHTTPCache::clear(). + */ + public function testClearMultiple() { + // Create a few cache entries. + $cid1 = $this->createCacheRecordUsingApi(); + $cid2 = $this->createCacheRecordUsingApi(); + $cid3 = $this->createCacheRecordUsingApi(); + $cid4 = $this->createCacheRecordUsingApi(); + + // Remove item 2 and 4. + $this->cache->clear(array( + $cid2, + $cid4, + )); + + // Assert that some records are removed. + $this->assertCacheItemExists($cid1); + $this->assertNoCacheItemExists($cid2); + $this->assertCacheItemExists($cid3); + $this->assertNoCacheItemExists($cid4); + + // Assert that only these files were removed. + $this->assertCacheFileExists($cid1); + $this->assertNoCacheFileExists($cid2); + $this->assertCacheFileExists($cid3); + $this->assertNoCacheFileExists($cid4); + } + + /** + * @covers FeedsHTTPCache::clear(). + */ + public function testClearAll() { + // Create a few cache entries. + $cid1 = $this->createCacheRecordUsingApi(); + $cid2 = $this->createCacheRecordUsingApi(); + $cid3 = $this->createCacheRecordUsingApi(); + + // Now clear complete cache. + $this->cache->clear('*', TRUE); + + // Assert that cache_feeds_http is empty. + $count = db_select('cache_feeds_http') + ->fields('cache_feeds_http', array('cid')) + ->countQuery() + ->execute() + ->fetchField(); + $this->assertEqual(0, $count, 'The cache_feeds_http item is empty.'); + + // Assert that the feeds cache dir is empty. + $empty = (count(glob(static::FEEDS_CACHE_DIR . '/*')) === 0); + $this->assertTrue($empty, 'The feeds cache directory is empty.'); + } + + /** + * @covers FeedsHTTPCache::clear(). + */ + public function testClearUsingWildcard() { + // Create a few cids, where the first few chars of cid1 and cid3 are the + // same and cid2 has a completely different string. + $cid1 = 'abc123'; + $cid2 = 'def456'; + $cid3 = 'abc789'; + $this->createCacheRecordUsingApi($cid1); + $this->createCacheRecordUsingApi($cid2); + $this->createCacheRecordUsingApi($cid3); + + // Clear all cache entries that start with 'abc'. + $this->cache->clear('abc', TRUE); + + // Assert that all records starting with 'abc' are gone. + $this->assertNoCacheItemExists($cid1); + $this->assertCacheItemExists($cid2); + $this->assertNoCacheItemExists($cid3); + + // Assert that all files in the cache dir starting with 'abc' are gone. + $this->assertNoCacheFileExists($cid1); + $this->assertCacheFileExists($cid2); + $this->assertNoCacheFileExists($cid3); + } + + /** + * @covers FeedsHTTPCache::isEmpty(). + */ + public function testIsEmpty() { + // Add a record to the cache_feeds_http table. + $this->createCacheRecord(); + + // Assert that the cache is not empty. + $this->assertFalse($this->cache->isEmpty(), 'The cache is empty.'); + + // Truncate the table and assert that the cache class tells us that it is + // empty. + db_truncate('cache_feeds_http')->execute(); + $this->assertTrue($this->cache->isEmpty()); + + // Add a file to the cache dir, without a entry in the database. + $dir = static::FEEDS_CACHE_DIR; + file_prepare_directory($dir, FILE_CREATE_DIRECTORY); + file_put_contents(static::FEEDS_CACHE_DIR . '/abc123', static::randomName()); + // And assert that the cache class reports that the cache is not empty. + $this->assertFalse($this->cache->isEmpty()); + } +}