diff --git a/core/modules/aggregator/aggregator.admin.inc b/core/modules/aggregator/aggregator.admin.inc index 632f1ec..4189f7c 100644 --- a/core/modules/aggregator/aggregator.admin.inc +++ b/core/modules/aggregator/aggregator.admin.inc @@ -341,70 +341,50 @@ function aggregator_admin_form($form, $form_state) { '#description' => t('A space-separated list of HTML tags allowed in the content of feed items. Disallowed tags are stripped from the content.'), ); - // Make sure configuration is sane. - aggregator_sanitize_configuration(); - - // Get all available fetchers. - $fetcher_manager = drupal_container()->get('plugin.manager.aggregator.fetcher'); - $fetchers = array(); - foreach ($fetcher_manager->getDefinitions() as $id => $definition) { - $label = $definition['title'] . ' ' . $definition['description'] . ''; - $fetchers[$id] = $label; - } + $config = config('aggregator.settings'); - // Get all available parsers. - $parsers = module_implements('aggregator_parse'); - foreach ($parsers as $k => $module) { - if ($info = module_invoke($module, 'aggregator_parse_info')) { - $label = $info['title'] . ' ' . $info['description'] . ''; - } - else { - $label = $module; + // Get all available fetchers, parsers and processors. + foreach (array('fetcher', 'parser', 'processor') as $type) { + // Initialize definitions if not set. + $definitions[$type] = isset($definitions[$type]) ? $definitions[$type] : array(); + $managers[$type] = drupal_container()->get("plugin.manager.aggregator.$type"); + foreach ($managers[$type]->getDefinitions() as $id => $definition) { + $label = $definition['title'] . ' ' . $definition['description'] . ''; + $definitions[$type][$id] = $label; } - unset($parsers[$k]); - $parsers[$module] = $label; } - // Get all available processors. - $processors = module_implements('aggregator_process'); - foreach ($processors as $k => $module) { - if ($info = module_invoke($module, 'aggregator_process_info')) { - $label = $info['title'] . ' ' . $info['description'] . ''; - } - else { - $label = $module; - } - unset($processors[$k]); - $processors[$module] = $label; - } + // Store definitions and managers so we can access them later. + $form['#definitions'] = $definitions; + $form['#managers'] = $managers; // Only show basic configuration if there are actually options. $basic_conf = array(); - if (count($fetchers) > 1) { + if (count($definitions['fetcher']) > 1) { $basic_conf['aggregator_fetcher'] = array( '#type' => 'radios', '#title' => t('Fetcher'), '#description' => t('Fetchers download data from an external source. Choose a fetcher suitable for the external source you would like to download from.'), - '#options' => $fetchers, - '#default_value' => config('aggregator.settings')->get('fetcher'), + '#options' => $definitions['fetcher'], + '#default_value' => $config->get('fetcher'), ); } - if (count($parsers) > 1) { + if (count($definitions['parser']) > 1) { $basic_conf['aggregator_parser'] = array( '#type' => 'radios', '#title' => t('Parser'), '#description' => t('Parsers transform downloaded data into standard structures. Choose a parser suitable for the type of feeds you would like to aggregate.'), - '#options' => $parsers, - '#default_value' => config('aggregator.settings')->get('parser'), + '#options' => $definitions['parser'], + '#default_value' => $config->get('parser'), ); } - if (count($processors) > 1) { + if (count($definitions['processor']) > 1) { $basic_conf['aggregator_processors'] = array( '#type' => 'checkboxes', '#title' => t('Processors'), '#description' => t('Processors act on parsed feed data, for example they store feed items. Choose the processors suitable for your task.'), - '#options' => $processors, - '#default_value' => config('aggregator.settings')->get('processors'), + '#options' => $definitions['processor'], + '#default_value' => $config->get('processors'), ); } if (count($basic_conf)) { @@ -417,9 +397,14 @@ function aggregator_admin_form($form, $form_state) { $form['basic_conf'] += $basic_conf; } - // Implementing modules will expect an array at $form['modules']. - $form['modules'] = array(); - + // Implementing processor plugins will expect an array at $form['processors']. + $form['processors'] = array(); + // Call settingsForm() for each acrive processor. + foreach ($definitions['processor'] as $id => $definition) { + if (in_array($id, $config->get('processors'))) { + $form = $managers['processor']->createInstance($id)->settingsForm($form, $form_state); + } + } return system_config_form($form, $form_state); } @@ -428,13 +413,14 @@ function aggregator_admin_form($form, $form_state) { */ function aggregator_admin_form_submit($form, &$form_state) { $config = config('aggregator.settings'); - $config - ->set('items.allowed_html', $form_state['values']['aggregator_allowed_html_tags']) - ->set('items.expire', $form_state['values']['aggregator_clear']) - ->set('items.teaser_length', $form_state['values']['aggregator_teaser_length']) - ->set('source.list_max', $form_state['values']['aggregator_summary_items']) - ->set('source.category_selector', $form_state['values']['aggregator_category_selector']); + // Let active processors save their settings. + foreach ($form['#definitions']['processor'] as $id => $definition) { + if (in_array($id, $config->get('processors'))) { + $form['#managers']['processor']->createInstance($id)->settingsSubmit($form, $form_state); + } + } + $config->set('items.allowed_html', $form_state['values']['aggregator_allowed_html_tags']); if (isset($form_state['values']['aggregator_fetcher'])) { $config->set('fetcher', $form_state['values']['aggregator_fetcher']); } diff --git a/core/modules/aggregator/aggregator.api.php b/core/modules/aggregator/aggregator.api.php deleted file mode 100644 index a6cb1c5..0000000 --- a/core/modules/aggregator/aggregator.api.php +++ /dev/null @@ -1,192 +0,0 @@ - array( - 'class' => 'Drupal\aggregator\Plugin\aggregator\fetcher\DefaultFetcher', - 'title' => t('Default fetcher'), - 'description' => t('Downloads data from a URL using Drupal\'s HTTP request handler.'), - ), - ); -} - -/** - * Create an alternative parser for aggregator module. - * - * A parser converts feed item data to a common format. The parser is called - * at the second of the three aggregation stages: first, data is downloaded - * by the active fetcher; second, it is converted to a common format by the - * active parser; and finally, it is passed to all active processors which - * manipulate or store the data. - * - * Modules that define this hook can be set as the active parser within the - * configuration page. Only one parser can be active at a time. - * - * @param $feed - * An object describing the resource to be parsed. $feed->source_string - * contains the raw feed data. The hook implementation should parse this data - * and add the following properties to the $feed object: - * - description: The human-readable description of the feed. - * - link: A full URL that directly relates to the feed. - * - image: An image URL used to display an image of the feed. - * - etag: An entity tag from the HTTP header used for cache validation to - * determine if the content has been changed. - * - modified: The UNIX timestamp when the feed was last modified. - * - items: An array of feed items. The common format for a single feed item - * is an associative array containing: - * - title: The human-readable title of the feed item. - * - description: The full body text of the item or a summary. - * - timestamp: The UNIX timestamp when the feed item was last published. - * - author: The author of the feed item. - * - guid: The global unique identifier (GUID) string that uniquely - * identifies the item. If not available, the link is used to identify - * the item. - * - link: A full URL to the individual feed item. - * - * @return - * TRUE if parsing was successful, FALSE otherwise. - * - * @see hook_aggregator_parse_info() - * @see hook_aggregator_fetch() - * @see hook_aggregator_process() - * - * @ingroup aggregator - */ -function hook_aggregator_parse($feed) { - if ($items = mymodule_parse($feed->source_string)) { - $feed->items = $items; - return TRUE; - } - return FALSE; -} - -/** - * Specify the title and short description of your parser. - * - * The title and the description provided are shown within the configuration - * page. Use as title the human readable name of the parser and as description - * a brief (40 to 80 characters) explanation of the parser's functionality. - * - * This hook is only called if your module implements hook_aggregator_parse(). - * If this hook is not implemented aggregator will use your module's file name - * as title and there will be no description. - * - * @return - * An associative array defining a title and a description string. - * - * @see hook_aggregator_parse() - * - * @ingroup aggregator - */ -function hook_aggregator_parse_info() { - return array( - 'title' => t('Default parser'), - 'description' => t('Default parser for RSS, Atom and RDF feeds.'), - ); -} - -/** - * Create a processor for aggregator.module. - * - * A processor acts on parsed feed data. Active processors are called at the - * third and last of the aggregation stages: first, data is downloaded by the - * active fetcher; second, it is converted to a common format by the active - * parser; and finally, it is passed to all active processors that manipulate or - * store the data. - * - * Modules that define this hook can be activated as a processor within the - * configuration page. - * - * @param $feed - * A feed object representing the resource to be processed. $feed->items - * contains an array of feed items downloaded and parsed at the parsing stage. - * See hook_aggregator_parse() for the basic format of a single item in the - * $feed->items array. For the exact format refer to the particular parser in - * use. - * - * @see hook_aggregator_process_info() - * @see hook_aggregator_fetch() - * @see hook_aggregator_parse() - * - * @ingroup aggregator - */ -function hook_aggregator_process($feed) { - foreach ($feed->items as $item) { - mymodule_save($item); - } -} - -/** - * Specify the title and short description of your processor. - * - * The title and the description provided are shown within the configuration - * page. Use as title the natural name of the processor and as description a - * brief (40 to 80 characters) explanation of the functionality. - * - * This hook is only called if your module implements hook_aggregator_process(). - * If this hook is not implemented aggregator will use your module's file name - * as title and there will be no description. - * - * @return - * An associative array defining a title and a description string. - * - * @see hook_aggregator_process() - * - * @ingroup aggregator - */ -function hook_aggregator_process_info($feed) { - return array( - 'title' => t('Default processor'), - 'description' => t('Creates lightweight records of feed items.'), - ); -} - -/** - * Remove stored feed data. - * - * Aggregator calls this hook if either a feed is deleted or a user clicks on - * "remove items". - * - * If your module stores feed items for example on hook_aggregator_process() it - * is recommended to implement this hook and to remove data related to $feed - * when called. - * - * @param $feed - * The $feed object whose items are being removed. - * - * @ingroup aggregator - */ -function hook_aggregator_remove($feed) { - mymodule_remove_items($feed->fid); -} - -/** - * @} End of "addtogroup hooks". - */ diff --git a/core/modules/aggregator/aggregator.module b/core/modules/aggregator/aggregator.module index 6e89fde..927162e 100644 --- a/core/modules/aggregator/aggregator.module +++ b/core/modules/aggregator/aggregator.module @@ -6,6 +6,7 @@ */ use Drupal\aggregator\Plugin\Core\Entity\Feed; +use Drupal\Component\Plugin\Exception\PluginException; /** * Denotes that a feed's items should never expire. @@ -304,7 +305,7 @@ function aggregator_permission() { * Queues news feeds for updates once their refresh interval has elapsed. */ function aggregator_cron() { - $result = db_query('SELECT fid FROM {aggregator_feed} WHERE queued = 0 AND checked + refresh < :time AND refresh <> :never', array( + $result = db_query('SELECT fid FROM {aggregator_feed} WHERE queued = 0 AND checked + refresh < :time AND refresh <> :never', array( ':time' => REQUEST_TIME, ':never' => AGGREGATOR_CLEAR_NEVER )); @@ -313,18 +314,22 @@ function aggregator_cron() { $feed = aggregator_feed_load($fid); if ($queue->createItem($feed)) { // Add timestamp to avoid queueing item more than once. - db_update('aggregator_feed') - ->fields(array('queued' => REQUEST_TIME)) - ->condition('fid', $feed->id()) - ->execute(); + $feed->queued->value = REQUEST_TIME; + $feed->save(); } } // Remove queued timestamp after 6 hours assuming the update has failed. - db_update('aggregator_feed') - ->fields(array('queued' => 0)) + $result = entity_query('aggregator_feed') ->condition('queued', REQUEST_TIME - (3600 * 6), '<') ->execute(); + if ($result) { + $feeds = entity_load_multiple('aggregator_feed', $result); + foreach ($feeds as $feed) { + $feed->queued->value = 0; + $feed->save(); + } + } } /** @@ -398,42 +403,23 @@ function aggregator_save_category($edit) { * An object describing the feed to be cleared. */ function aggregator_remove(Feed $feed) { - _aggregator_get_variables(); - // Call hook_aggregator_remove() on all modules. - module_invoke_all('aggregator_remove', $feed); - // Reset feed. - db_update('aggregator_feed') - ->condition('fid', $feed->id()) - ->fields(array( - 'checked' => 0, - 'hash' => '', - 'etag' => '', - 'modified' => 0, - )) - ->execute(); -} - -/** - * Gets the fetcher, parser, and processors. - * - * @return - * An array containing the fetcher, parser, and processors. - */ -function _aggregator_get_variables() { - $config = config('aggregator.settings'); - $fetcher = $config->get('fetcher'); - - $parser = $config->get('parser'); - if ($parser == 'aggregator') { - include_once DRUPAL_ROOT . '/' . drupal_get_path('module', 'aggregator') . '/aggregator.parser.inc'; - } - - $processors = $config->get('processors'); - if (in_array('aggregator', $processors)) { - include_once DRUPAL_ROOT . '/' . drupal_get_path('module', 'aggregator') . '/aggregator.processor.inc'; + // Call ProcessorInterface::remove() on all processors. + $manager = drupal_container()->get('plugin.manager.aggregator.processor'); + foreach ($manager->getDefinitions() as $id => $definition) { + try { + $manager->createInstance($id)->remove($feed); + } + catch (PluginException $e) { + // Fail silently + // @todo Maybe add a watchdog entry? + } } - - return array($fetcher, $parser, $processors); + // Reset feed. + $feed->checked->value = 0; + $feed->hash->value = ''; + $feed->etag->value = ''; + $feed->modified->value = 0; + $feed->save(); } /** @@ -446,15 +432,29 @@ function aggregator_refresh(Feed $feed) { // Store feed URL to track changes. $feed_url = $feed->url->value; - list($fetcher, $parser, $processors) = _aggregator_get_variables(); - + $config = config('aggregator.settings'); // Fetch the feed. $fetcher_manager = drupal_container()->get('plugin.manager.aggregator.fetcher'); try { - $success = $fetcher_manager->createInstance($fetcher)->fetch($feed); + $success = $fetcher_manager->createInstance($config->get('fetcher'))->fetch($feed); } catch (PluginException $e) { $success = FALSE; + // @todo Maybe add a watchdog entry? + } + + // Retrieve processor manager now. + $processor_manager = drupal_container()->get('plugin.manager.aggregator.processor'); + // Store instances in an array so we dont have to instantiate new objects. + $processor_instances = array(); + foreach ($config->get('processors') as $processor) { + try { + $processor_instances[$processor] = $processor_manager->createInstance($processor); + } + catch (PluginException $e) { + // Fail silently + // @todo Maybe add a watchdog entry? + } } // We store the hash of feed data in the database. When refreshing a @@ -464,43 +464,49 @@ function aggregator_refresh(Feed $feed) { if ($success && ($feed->hash->value != $hash)) { // Parse the feed. - if (module_invoke($parser, 'aggregator_parse', $feed)) { - if (empty($feed->link->value)) { - $feed->link->value = $feed->url->value; - } - $feed->hash->value = $hash; - // Update feed with parsed data. - $feed->save(); + $parser_manager = drupal_container()->get('plugin.manager.aggregator.parser'); + try { + if ($parser_manager->createInstance($config->get('parser'))->parse($feed)) { + if (empty($feed->link->value)) { + $feed->link->value = $feed->url->value; + } + $feed->hash->value = $hash; + // Update feed with parsed data. + $feed->save(); - // Log if feed URL has changed. - if ($feed->url->value != $feed_url) { - watchdog('aggregator', 'Updated URL for feed %title to %url.', array('%title' => $feed->label(), '%url' => $feed->url->value)); - } + // Log if feed URL has changed. + if ($feed->url->value != $feed_url) { + watchdog('aggregator', 'Updated URL for feed %title to %url.', array('%title' => $feed->label(), '%url' => $feed->url->value)); + } - watchdog('aggregator', 'There is new syndicated content from %site.', array('%site' => $feed->label())); - drupal_set_message(t('There is new syndicated content from %site.', array('%site' => $feed->label()))); + watchdog('aggregator', 'There is new syndicated content from %site.', array('%site' => $feed->label())); + drupal_set_message(t('There is new syndicated content from %site.', array('%site' => $feed->label()))); - // If there are items on the feed, let all enabled processors do their work on it. - if (@count($feed->items)) { - foreach ($processors as $processor) { - module_invoke($processor, 'aggregator_process', $feed); + // If there are items on the feed, let all enabled processors do their work on it. + if (@count($feed->items)) { + foreach ($processor_instances as $instance) { + $instance->process($feed); + } } } } + catch (PluginException $e) { + // Fail silently + // @todo Maybe add a watchdog entry? + } } else { drupal_set_message(t('There is no new syndicated content from %site.', array('%site' => $feed->label()))); } // Regardless of successful or not, indicate that this feed has been checked. - db_update('aggregator_feed') - ->fields(array('checked' => REQUEST_TIME, 'queued' => 0)) - ->condition('fid', $feed->id()) - ->execute(); + $feed->checked->value = REQUEST_TIME; + $feed->queued->value = 0; + $feed->save(); - // Expire old feed items. - if (function_exists('aggregator_expire')) { - aggregator_expire($feed); + // Processing is done call postProcess on enabled processors. + foreach ($processor_instances as $instance) { + $instance->postProcess($feed); } } @@ -564,55 +570,6 @@ function aggregator_filter_xss($value) { } /** - * Checks and sanitizes the aggregator configuration. - * - * Goes through all fetchers, parsers and processors and checks whether they - * are available. If one is missing, resets to standard configuration. - * - * @return - * TRUE if this function resets the configuration; FALSE if not. - */ -function aggregator_sanitize_configuration() { - $reset = FALSE; - list($fetcher, $parser, $processors) = _aggregator_get_variables(); - if (!module_exists($fetcher)) { - $reset = TRUE; - } - if (!module_exists($parser)) { - $reset = TRUE; - } - foreach ($processors as $processor) { - if (!module_exists($processor)) { - $reset = TRUE; - break; - } - } - if ($reset) { - // Reset aggregator config if necessary using the module defaults. - config('aggregator.settings') - ->set('fetcher', 'aggregator') - ->set('parser', 'aggregator') - ->set('processors', array('aggregator' => 'aggregator')) - ->save(); - return TRUE; - } - return FALSE; -} - -/** - * Helper function for drupal_map_assoc. - * - * @param $count - * Items count. - * - * @return - * A string that is plural-formatted as "@count items". - */ -function _aggregator_items($count) { - return format_plural($count, '1 item', '@count items'); -} - -/** * Implements hook_preprocess_HOOK() for block.tpl.php. */ function aggregator_preprocess_block(&$variables) { diff --git a/core/modules/aggregator/aggregator.parser.inc b/core/modules/aggregator/aggregator.parser.inc deleted file mode 100644 index 3b906e8..0000000 --- a/core/modules/aggregator/aggregator.parser.inc +++ /dev/null @@ -1,326 +0,0 @@ - t('Default parser'), - 'description' => t('Parses RSS, Atom and RDF feeds.'), - ); -} - -/** - * Implements hook_aggregator_parse(). - */ -function aggregator_aggregator_parse(Feed $feed) { - global $channel, $image; - - // Filter the input data. - if (aggregator_parse_feed($feed->source_string, $feed)) { - - // Prepare the channel data. - foreach ($channel as $key => $value) { - $channel[$key] = trim($value); - } - - // Prepare the image data (if any). - foreach ($image as $key => $value) { - $image[$key] = trim($value); - } - - // Add parsed data to the feed object. - $feed->link->value = !empty($channel['link']) ? $channel['link'] : ''; - $feed->description->value = !empty($channel['description']) ? $channel['description'] : ''; - $feed->image->value = !empty($image['url']) ? $image['url'] : ''; - - // Clear the page and block caches. - cache_invalidate_tags(array('content' => TRUE)); - - return TRUE; - } - - return FALSE; -} - -/** - * Parses a feed and stores its items. - * - * @param string $data - * The feed data. - * @param \Drupal\aggregator\Plugin\Core\Entity\Feed $feed - * An object describing the feed to be parsed. - * - * @return - * FALSE on error, TRUE otherwise. - */ -function aggregator_parse_feed(&$data, Feed $feed) { - global $items, $image, $channel; - - // Unset the global variables before we use them. - unset($GLOBALS['element'], $GLOBALS['item'], $GLOBALS['tag']); - $items = array(); - $image = array(); - $channel = array(); - - // Parse the data. - $xml_parser = drupal_xml_parser_create($data); - xml_set_element_handler($xml_parser, 'aggregator_element_start', 'aggregator_element_end'); - xml_set_character_data_handler($xml_parser, 'aggregator_element_data'); - - if (!xml_parse($xml_parser, $data, 1)) { - watchdog('aggregator', 'The feed from %site seems to be broken due to an error "%error" on line %line.', array('%site' => $feed->label(), '%error' => xml_error_string(xml_get_error_code($xml_parser)), '%line' => xml_get_current_line_number($xml_parser)), WATCHDOG_WARNING); - drupal_set_message(t('The feed from %site seems to be broken because of error "%error" on line %line.', array('%site' => $feed->label(), '%error' => xml_error_string(xml_get_error_code($xml_parser)), '%line' => xml_get_current_line_number($xml_parser))), 'error'); - return FALSE; - } - xml_parser_free($xml_parser); - - // We reverse the array such that we store the first item last, and the last - // item first. In the database, the newest item should be at the top. - $items = array_reverse($items); - - // Initialize items array. - $feed->items = array(); - foreach ($items as $item) { - - // Prepare the item: - foreach ($item as $key => $value) { - $item[$key] = trim($value); - } - - // Resolve the item's title. If no title is found, we use up to 40 - // characters of the description ending at a word boundary, but not - // splitting potential entities. - if (!empty($item['title'])) { - $item['title'] = $item['title']; - } - elseif (!empty($item['description'])) { - $item['title'] = preg_replace('/^(.*)[^\w;&].*?$/', "\\1", truncate_utf8($item['description'], 40)); - } - else { - $item['title'] = ''; - } - - // Resolve the items link. - if (!empty($item['link'])) { - $item['link'] = $item['link']; - } - else { - $item['link'] = $feed->link->value; - } - - // Atom feeds have an ID tag instead of a GUID tag. - if (!isset($item['guid'])) { - $item['guid'] = isset($item['id']) ? $item['id'] : ''; - } - - // Atom feeds have a content and/or summary tag instead of a description tag. - if (!empty($item['content:encoded'])) { - $item['description'] = $item['content:encoded']; - } - elseif (!empty($item['summary'])) { - $item['description'] = $item['summary']; - } - elseif (!empty($item['content'])) { - $item['description'] = $item['content']; - } - - // Try to resolve and parse the item's publication date. - $date = ''; - foreach (array('pubdate', 'dc:date', 'dcterms:issued', 'dcterms:created', 'dcterms:modified', 'issued', 'created', 'modified', 'published', 'updated') as $key) { - if (!empty($item[$key])) { - $date = $item[$key]; - break; - } - } - - $item['timestamp'] = strtotime($date); - - if ($item['timestamp'] === FALSE) { - $item['timestamp'] = aggregator_parse_w3cdtf($date); // Aggregator_parse_w3cdtf() returns FALSE on failure. - } - - // Resolve dc:creator tag as the item author if author tag is not set. - if (empty($item['author']) && !empty($item['dc:creator'])) { - $item['author'] = $item['dc:creator']; - } - - $item += array('author' => '', 'description' => ''); - - // Store on $feed object. This is where processors will look for parsed items. - $feed->items[] = $item; - } - - return TRUE; -} - -/** - * Performs an action when an opening tag is encountered. - * - * Callback function used by xml_parse() within aggregator_parse_feed(). - */ -function aggregator_element_start($parser, $name, $attributes) { - global $item, $element, $tag, $items, $channel; - - $name = strtolower($name); - switch ($name) { - case 'image': - case 'textinput': - case 'summary': - case 'tagline': - case 'subtitle': - case 'logo': - case 'info': - $element = $name; - break; - case 'id': - case 'content': - if ($element != 'item') { - $element = $name; - } - case 'link': - // According to RFC 4287, link elements in Atom feeds without a 'rel' - // attribute should be interpreted as though the relation type is - // "alternate". - if (!empty($attributes['HREF']) && (empty($attributes['REL']) || $attributes['REL'] == 'alternate')) { - if ($element == 'item') { - $items[$item]['link'] = $attributes['HREF']; - } - else { - $channel['link'] = $attributes['HREF']; - } - } - break; - case 'item': - $element = $name; - $item += 1; - break; - case 'entry': - $element = 'item'; - $item += 1; - break; - } - - $tag = $name; -} - -/** - * Performs an action when a closing tag is encountered. - * - * Callback function used by xml_parse() within aggregator_parse_feed(). - */ -function aggregator_element_end($parser, $name) { - global $element; - - switch ($name) { - case 'image': - case 'textinput': - case 'item': - case 'entry': - case 'info': - $element = ''; - break; - case 'id': - case 'content': - if ($element == $name) { - $element = ''; - } - } -} - -/** - * Performs an action when data is encountered. - * - * Callback function used by xml_parse() within aggregator_parse_feed(). - */ -function aggregator_element_data($parser, $data) { - global $channel, $element, $items, $item, $image, $tag; - $items += array($item => array()); - switch ($element) { - case 'item': - $items[$item] += array($tag => ''); - $items[$item][$tag] .= $data; - break; - case 'image': - case 'logo': - $image += array($tag => ''); - $image[$tag] .= $data; - break; - case 'link': - if ($data) { - $items[$item] += array($tag => ''); - $items[$item][$tag] .= $data; - } - break; - case 'content': - $items[$item] += array('content' => ''); - $items[$item]['content'] .= $data; - break; - case 'summary': - $items[$item] += array('summary' => ''); - $items[$item]['summary'] .= $data; - break; - case 'tagline': - case 'subtitle': - $channel += array('description' => ''); - $channel['description'] .= $data; - break; - case 'info': - case 'id': - case 'textinput': - // The sub-element is not supported. However, we must recognize - // it or its contents will end up in the item array. - break; - default: - $channel += array($tag => ''); - $channel[$tag] .= $data; - } -} - -/** - * Parses the W3C date/time format, a subset of ISO 8601. - * - * PHP date parsing functions do not handle this format. See - * http://www.w3.org/TR/NOTE-datetime for more information. Originally from - * MagpieRSS (http://magpierss.sourceforge.net/). - * - * @param $date_str - * A string with a potentially W3C DTF date. - * - * @return - * A timestamp if parsed successfully or FALSE if not. - */ -function aggregator_parse_w3cdtf($date_str) { - if (preg_match('/(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2})(:(\d{2}))?(?:([-+])(\d{2}):?(\d{2})|(Z))?/', $date_str, $match)) { - list($year, $month, $day, $hours, $minutes, $seconds) = array($match[1], $match[2], $match[3], $match[4], $match[5], $match[6]); - // Calculate the epoch for current date assuming GMT. - $epoch = gmmktime($hours, $minutes, $seconds, $month, $day, $year); - if ($match[10] != 'Z') { // Z is zulu time, aka GMT - list($tz_mod, $tz_hour, $tz_min) = array($match[8], $match[9], $match[10]); - // Zero out the variables. - if (!$tz_hour) { - $tz_hour = 0; - } - if (!$tz_min) { - $tz_min = 0; - } - $offset_secs = (($tz_hour * 60) + $tz_min) * 60; - // Is timezone ahead of GMT? If yes, subtract offset. - if ($tz_mod == '+') { - $offset_secs *= -1; - } - $epoch += $offset_secs; - } - return $epoch; - } - else { - return FALSE; - } -} diff --git a/core/modules/aggregator/aggregator.processor.inc b/core/modules/aggregator/aggregator.processor.inc deleted file mode 100644 index 10b52f1..0000000 --- a/core/modules/aggregator/aggregator.processor.inc +++ /dev/null @@ -1,182 +0,0 @@ - t('Default processor'), - 'description' => t('Creates lightweight records from feed items.'), - ); -} - -/** - * Implements hook_aggregator_process(). - */ -function aggregator_aggregator_process($feed) { - if (is_object($feed)) { - if (is_array($feed->items)) { - foreach ($feed->items as $item) { - // @todo: The default entity render controller always returns an empty - // array, which is ignored in aggregator_save_item() currently. Should - // probably be fixed. - if (empty($item['title'])) { - continue; - } - - // Save this item. Try to avoid duplicate entries as much as possible. If - // we find a duplicate entry, we resolve it and pass along its ID is such - // that we can update it if needed. - if (!empty($item['guid'])) { - $values = array('fid' => $feed->id(), 'guid' => $item['guid']); - } - elseif ($item['link'] && $item['link'] != $feed->link && $item['link'] != $feed->url) { - $values = array('fid' => $feed->id(), 'link' => $item['link']); - } - else { - $values = array('fid' => $feed->id(), 'title' => $item['title']); - } - - // Try to load an existing entry. - if ($entry = entity_load_multiple_by_properties('aggregator_item', $values)) { - $entry = reset($entry); - } - else { - $entry = entity_create('aggregator_item', array('langcode' => $feed->language()->langcode)); - } - if ($item['timestamp']) { - $entry->timestamp->value = $item['timestamp']; - } - - // Make sure the item title and author fit in the 255 varchar column. - $entry->title->value = truncate_utf8($item['title'], 255, TRUE, TRUE); - $entry->author->value = truncate_utf8($item['author'], 255, TRUE, TRUE); - - $entry->fid->value = $feed->id(); - $entry->link->value = $item['link']; - $entry->description->value = $item['description']; - $entry->guid->value = $item['guid']; - $entry->save(); - } - } - } -} - -/** - * Implements hook_aggregator_remove(). - */ -function aggregator_aggregator_remove($feed) { - $iids = db_query('SELECT iid FROM {aggregator_item} WHERE fid = :fid', array(':fid' => $feed->id()))->fetchCol(); - entity_delete_multiple('aggregator_item', $iids); - - drupal_set_message(t('The news items from %site have been removed.', array('%site' => $feed->label()))); -} - -/** - * Implements hook_form_aggregator_admin_form_alter(). - * - * Form alter aggregator module's own form to keep processor functionality - * separate from aggregator API functionality. - */ -function aggregator_form_aggregator_admin_form_alter(&$form, $form_state) { - $config = config('aggregator.settings'); - $aggregator_processors = $config->get('processors'); - if (in_array('aggregator', $aggregator_processors)) { - $info = module_invoke('aggregator', 'aggregator_process', 'info'); - $items = drupal_map_assoc(array(3, 5, 10, 15, 20, 25), '_aggregator_items'); - $period = drupal_map_assoc(array(3600, 10800, 21600, 32400, 43200, 86400, 172800, 259200, 604800, 1209600, 2419200, 4838400, 9676800), 'format_interval'); - $period[AGGREGATOR_CLEAR_NEVER] = t('Never'); - - // Only wrap into details if there is a basic configuration. - if (isset($form['basic_conf'])) { - $form['modules']['aggregator'] = array( - '#type' => 'details', - '#title' => t('Default processor settings'), - '#description' => $info['description'], - '#collapsed' => !in_array('aggregator', $aggregator_processors), - ); - } - else { - $form['modules']['aggregator'] = array(); - } - - $form['modules']['aggregator']['aggregator_summary_items'] = array( - '#type' => 'select', - '#title' => t('Number of items shown in listing pages'), - '#default_value' => config('aggregator.settings')->get('source.list_max'), - '#empty_value' => 0, - '#options' => $items, - ); - - $form['modules']['aggregator']['aggregator_clear'] = array( - '#type' => 'select', - '#title' => t('Discard items older than'), - '#default_value' => config('aggregator.settings')->get('items.expire'), - '#options' => $period, - '#description' => t('Requires a correctly configured cron maintenance task.', array('@cron' => url('admin/reports/status'))), - ); - - $form['modules']['aggregator']['aggregator_category_selector'] = array( - '#type' => 'radios', - '#title' => t('Select categories using'), - '#default_value' => config('aggregator.settings')->get('source.category_selector'), - '#options' => array('checkboxes' => t('checkboxes'), - 'select' => t('multiple selector')), - '#description' => t('For a small number of categories, checkboxes are easier to use, while a multiple selector works well with large numbers of categories.'), - ); - $form['modules']['aggregator']['aggregator_teaser_length'] = array( - '#type' => 'select', - '#title' => t('Length of trimmed description'), - '#default_value' => config('aggregator.settings')->get('items.teaser_length'), - '#options' => drupal_map_assoc(array(0, 200, 400, 600, 800, 1000, 1200, 1400, 1600, 1800, 2000), '_aggregator_characters'), - '#description' => t("The maximum number of characters used in the trimmed version of content.") - ); - - } -} - -/** - * Creates display text for teaser length option values. - * - * Callback for drupal_map_assoc() within - * aggregator_form_aggregator_admin_form_alter(). - * - * @param int $length - * The desired length of teaser text, in bytes. - * - * @return string - * A translated string explaining the teaser string length. - */ -function _aggregator_characters($length) { - return ($length == 0) ? t('Unlimited') : format_plural($length, '1 character', '@count characters'); -} - -/** - * Expires items from a feed depending on expiration settings. - * - * @param \Drupal\aggregator\Plugin\Core\Entity\Feed $feed - * Object describing feed. - */ -function aggregator_expire(Feed $feed) { - $aggregator_clear = config('aggregator.settings')->get('items.expire'); - - if ($aggregator_clear != AGGREGATOR_CLEAR_NEVER) { - // Remove all items that are older than flush item timer. - $age = REQUEST_TIME - $aggregator_clear; - $iids = db_query('SELECT iid FROM {aggregator_item} WHERE fid = :fid AND timestamp < :timestamp', array( - ':fid' => $feed->id(), - ':timestamp' => $age, - )) - ->fetchCol(); - if ($iids) { - entity_delete_multiple('aggregator_item', $iids); - } - } -} diff --git a/core/modules/aggregator/lib/Drupal/aggregator/AggregatorBundle.php b/core/modules/aggregator/lib/Drupal/aggregator/AggregatorBundle.php index 9c97d9a..2782029 100644 --- a/core/modules/aggregator/lib/Drupal/aggregator/AggregatorBundle.php +++ b/core/modules/aggregator/lib/Drupal/aggregator/AggregatorBundle.php @@ -19,8 +19,11 @@ class AggregatorBundle extends Bundle { * Overrides Bundle::build(). */ public function build(ContainerBuilder $container) { - $container->register('plugin.manager.aggregator.fetcher', 'Drupal\aggregator\Plugin\FetcherManager') - ->addArgument('%container.namespaces%'); + foreach (array('fetcher', 'parser', 'processor') as $type) { + $container->register("plugin.manager.aggregator.$type", 'Drupal\aggregator\Plugin\AggregatorPluginManager') + ->addArgument($type) + ->addArgument('%container.namespaces%'); + } } } diff --git a/core/modules/aggregator/lib/Drupal/aggregator/FeedStorageController.php b/core/modules/aggregator/lib/Drupal/aggregator/FeedStorageController.php index d8c052c..e9a738b 100644 --- a/core/modules/aggregator/lib/Drupal/aggregator/FeedStorageController.php +++ b/core/modules/aggregator/lib/Drupal/aggregator/FeedStorageController.php @@ -51,9 +51,10 @@ protected function preDelete($entities) { drupal_container()->get('plugin.manager.block')->clearCachedDefinitions(); } foreach ($entities as $entity) { - $iids = db_query('SELECT iid FROM {aggregator_item} WHERE fid = :fid', array(':fid' => $entity->id()))->fetchCol(); - if ($iids) { - entity_delete_multiple('aggregator_item', $iids); + // Notify processors to remove stored items. + $manager = drupal_container()->get('plugin.manager.aggregator.processor'); + foreach ($manager->getDefinitions() as $id => $definition) { + $manager->createInstance($id)->remove($entity); } } } diff --git a/core/modules/aggregator/lib/Drupal/aggregator/Plugin/FetcherManager.php b/core/modules/aggregator/lib/Drupal/aggregator/Plugin/AggregatorPluginManager.php similarity index 51% rename from core/modules/aggregator/lib/Drupal/aggregator/Plugin/FetcherManager.php rename to core/modules/aggregator/lib/Drupal/aggregator/Plugin/AggregatorPluginManager.php index be8fc79..33c5988 100644 --- a/core/modules/aggregator/lib/Drupal/aggregator/Plugin/FetcherManager.php +++ b/core/modules/aggregator/lib/Drupal/aggregator/Plugin/AggregatorPluginManager.php @@ -2,7 +2,7 @@ /** * @file - * Definition of Drupal\aggregator\Plugin\FetcherManager. + * Definition of Drupal\aggregator\Plugin\AggregatorPluginManager. */ namespace Drupal\aggregator\Plugin; @@ -13,19 +13,21 @@ use Drupal\Core\Plugin\Discovery\CacheDecorator; /** - * Manages aggregator fetcher plugins. + * Manages aggregator plugins. */ -class FetcherManager extends PluginManagerBase { +class AggregatorPluginManager extends PluginManagerBase { /** - * Constructs a FetcherManager object. + * Constructs a AggregatorPluginManager object. * + * @param string $type + * The plugin type, for example fetcher. * @param array $namespaces * An array of paths keyed by it's corresponding namespaces. */ - public function __construct(array $namespaces) { - $this->discovery = new AnnotatedClassDiscovery('aggregator', 'fetcher', $namespaces); - $this->discovery = new CacheDecorator($this->discovery, 'aggregator_fetcher:' . language(LANGUAGE_TYPE_INTERFACE)->langcode); + public function __construct($type, array $namespaces) { + $this->discovery = new AnnotatedClassDiscovery('aggregator', $type, $namespaces); + $this->discovery = new CacheDecorator($this->discovery, "aggregator_$type:" . language(LANGUAGE_TYPE_INTERFACE)->langcode); $this->factory = new DefaultFactory($this->discovery); } } diff --git a/core/modules/aggregator/lib/Drupal/aggregator/Plugin/FetcherInterface.php b/core/modules/aggregator/lib/Drupal/aggregator/Plugin/FetcherInterface.php index 120f596..da1377d 100644 --- a/core/modules/aggregator/lib/Drupal/aggregator/Plugin/FetcherInterface.php +++ b/core/modules/aggregator/lib/Drupal/aggregator/Plugin/FetcherInterface.php @@ -2,7 +2,7 @@ /** * @file - * Definition of Drupal\aggregator\Plugin\FetcherInterface. + * Contains \Drupal\aggregator\Plugin\FetcherInterface. */ namespace Drupal\aggregator\Plugin; diff --git a/core/modules/aggregator/lib/Drupal/aggregator/Plugin/ParserInterface.php b/core/modules/aggregator/lib/Drupal/aggregator/Plugin/ParserInterface.php new file mode 100644 index 0000000..2b49663 --- /dev/null +++ b/core/modules/aggregator/lib/Drupal/aggregator/Plugin/ParserInterface.php @@ -0,0 +1,53 @@ +source_string->value contains the raw feed data. Parse the data + * and add the following properties to the $feed object: + * - description: The human-readable description of the feed. + * - link: A full URL that directly relates to the feed. + * - image: An image URL used to display an image of the feed. + * - etag: An entity tag from the HTTP header used for cache validation to + * determine if the content has been changed. + * - modified: The UNIX timestamp when the feed was last modified. + * - items: An array of feed items. The common format for a single feed item + * is an associative array containing: + * - title: The human-readable title of the feed item. + * - description: The full body text of the item or a summary. + * - timestamp: The UNIX timestamp when the feed item was last published. + * - author: The author of the feed item. + * - guid: The global unique identifier (GUID) string that uniquely + * identifies the item. If not available, the link is used to identify + * the item. + * - link: A full URL to the individual feed item. + * + * @return + * TRUE if parsing was successful, FALSE otherwise. + */ + public function parse(Feed $feed); + +} diff --git a/core/modules/aggregator/lib/Drupal/aggregator/Plugin/ProcessorInterface.php b/core/modules/aggregator/lib/Drupal/aggregator/Plugin/ProcessorInterface.php new file mode 100644 index 0000000..568e385 --- /dev/null +++ b/core/modules/aggregator/lib/Drupal/aggregator/Plugin/ProcessorInterface.php @@ -0,0 +1,86 @@ +items + * contains an array of feed items downloaded and parsed at the parsing stage. + * See Drupal\aggregator\Plugin\FetcherInterface::parse() + * for the basic format of a single item in the $feed->items array. + * For the exact format refer to the particular parser in use. + * + */ + public function process(Feed $feed); + + /** + * Refresh feed information. + * + * Called after the processing of the feed is completed + * by all selected processors. + * + * @param \Drupal\aggregator\Plugin\Core\Entity\Feed $feed + * Object describing feed. + * + * @see aggregator_refresh() + */ + public function postProcess(Feed $feed); + + /** + * Remove stored feed data. + * + * Called by aggregator if either a feed is deleted or a user clicks on + * "remove items". + * + * @param \Drupal\aggregator\Plugin\Core\Entity\Feed $feed + * The $feed object whose items are being removed. + * + */ + public function remove(Feed $feed); + +} diff --git a/core/modules/aggregator/lib/Drupal/aggregator/Plugin/aggregator/fetcher/DefaultFetcher.php b/core/modules/aggregator/lib/Drupal/aggregator/Plugin/aggregator/fetcher/DefaultFetcher.php index 4775873..7e1d117 100644 --- a/core/modules/aggregator/lib/Drupal/aggregator/Plugin/aggregator/fetcher/DefaultFetcher.php +++ b/core/modules/aggregator/lib/Drupal/aggregator/Plugin/aggregator/fetcher/DefaultFetcher.php @@ -2,7 +2,7 @@ /** * @file - * Definition of Drupal\aggregator\Plugin\aggregator\fetcher\DefaultFetcher. + * Contains \Drupal\aggregator\Plugin\aggregator\fetcher\DefaultFetcher. */ namespace Drupal\aggregator\Plugin\aggregator\fetcher; @@ -16,7 +16,7 @@ /** * Defines a default fetcher implementation. * - * Uses drupal_http_request() to download the feed. + * Uses http_default_client class to download the feed. * * @Plugin( * id = "aggregator", @@ -27,9 +27,9 @@ class DefaultFetcher implements FetcherInterface { /** - * Implements Drupal\aggregator\Plugin\FetcherInterface::fetch(). + * Implements \Drupal\aggregator\Plugin\FetcherInterface::fetch(). */ - function fetch(Feed $feed) { + public function fetch(Feed $feed) { $request = drupal_container()->get('http_default_client')->get($feed->url->value); $feed->source_string = FALSE; diff --git a/core/modules/aggregator/lib/Drupal/aggregator/Plugin/aggregator/parser/DefaultParser.php b/core/modules/aggregator/lib/Drupal/aggregator/Plugin/aggregator/parser/DefaultParser.php new file mode 100644 index 0000000..f8b3639 --- /dev/null +++ b/core/modules/aggregator/lib/Drupal/aggregator/Plugin/aggregator/parser/DefaultParser.php @@ -0,0 +1,351 @@ +parseFeed($feed->source_string, $feed)) { + + // Prepare the channel data. + foreach ($this->channel as $key => $value) { + $this->channel[$key] = trim($value); + } + + // Prepare the image data (if any). + foreach ($this->image as $key => $value) { + $this->image[$key] = trim($value); + } + + // Add parsed data to the feed object. + $feed->link->value = !empty($channel['link']) ? $channel['link'] : ''; + $feed->description->value = !empty($channel['description']) ? $channel['description'] : ''; + $feed->image->value = !empty($image['url']) ? $image['url'] : ''; + + // Clear the page and block caches. + cache_invalidate_tags(array('content' => TRUE)); + + return TRUE; + } + + return FALSE; + } + + + /** + * Parses a feed and stores its items. + * + * @param string $data + * The feed data. + * @param \Drupal\aggregator\Plugin\Core\Entity\Feed $feed + * An object describing the feed to be parsed. + * + * @return + * FALSE on error, TRUE otherwise. + */ + protected function parseFeed(&$data, Feed $feed) { + // Parse the data. + $xml_parser = drupal_xml_parser_create($data); + xml_set_element_handler($xml_parser, array($this, 'elementStart'), array($this, 'elementEnd')); + xml_set_character_data_handler($xml_parser, array($this, 'elementData')); + + if (!xml_parse($xml_parser, $data, 1)) { + watchdog('aggregator', 'The feed from %site seems to be broken due to an error "%error" on line %line.', array('%site' => $feed->label(), '%error' => xml_error_string(xml_get_error_code($xml_parser)), '%line' => xml_get_current_line_number($xml_parser)), WATCHDOG_WARNING); + drupal_set_message(t('The feed from %site seems to be broken because of error "%error" on line %line.', array('%site' => $feed->label(), '%error' => xml_error_string(xml_get_error_code($xml_parser)), '%line' => xml_get_current_line_number($xml_parser))), 'error'); + return FALSE; + } + xml_parser_free($xml_parser); + + // We reverse the array such that we store the first item last, and the last + // item first. In the database, the newest item should be at the top. + $this->items = array_reverse($this->items); + + // Initialize items array. + $feed->items = array(); + foreach ($this->items as $item) { + + // Prepare the item: + foreach ($item as $key => $value) { + $item[$key] = trim($value); + } + + // Resolve the item's title. If no title is found, we use up to 40 + // characters of the description ending at a word boundary, but not + // splitting potential entities. + if (!empty($item['title'])) { + $item['title'] = $item['title']; + } + elseif (!empty($item['description'])) { + $item['title'] = preg_replace('/^(.*)[^\w;&].*?$/', "\\1", truncate_utf8($item['description'], 40)); + } + else { + $item['title'] = ''; + } + + // Resolve the items link. + if (!empty($item['link'])) { + $item['link'] = $item['link']; + } + else { + $item['link'] = $feed->link->value; + } + + // Atom feeds have an ID tag instead of a GUID tag. + if (!isset($item['guid'])) { + $item['guid'] = isset($item['id']) ? $item['id'] : ''; + } + + // Atom feeds have a content and/or summary tag instead of a description tag. + if (!empty($item['content:encoded'])) { + $item['description'] = $item['content:encoded']; + } + elseif (!empty($item['summary'])) { + $item['description'] = $item['summary']; + } + elseif (!empty($item['content'])) { + $item['description'] = $item['content']; + } + + // Try to resolve and parse the item's publication date. + $date = ''; + foreach (array('pubdate', 'dc:date', 'dcterms:issued', 'dcterms:created', 'dcterms:modified', 'issued', 'created', 'modified', 'published', 'updated') as $key) { + if (!empty($item[$key])) { + $date = $item[$key]; + break; + } + } + + $item['timestamp'] = strtotime($date); + + if ($item['timestamp'] === FALSE) { + $item['timestamp'] = $this->parseW3cdtf($date); // Aggregator_parse_w3cdtf() returns FALSE on failure. + } + + // Resolve dc:creator tag as the item author if author tag is not set. + if (empty($item['author']) && !empty($item['dc:creator'])) { + $item['author'] = $item['dc:creator']; + } + + $item += array('author' => '', 'description' => ''); + + // Store on $feed object. This is where processors will look for parsed items. + $feed->items[] = $item; + } + + return TRUE; + } + + /** + * Performs an action when an opening tag is encountered. + */ + protected function elementStart($parser, $name, $attributes) { + $name = strtolower($name); + switch ($name) { + case 'image': + case 'textinput': + case 'summary': + case 'tagline': + case 'subtitle': + case 'logo': + case 'info': + $this->element = $name; + break; + case 'id': + case 'content': + if ($this->element != 'item') { + $this->element = $name; + } + case 'link': + // According to RFC 4287, link elements in Atom feeds without a 'rel' + // attribute should be interpreted as though the relation type is + // "alternate". + if (!empty($attributes['HREF']) && (empty($attributes['REL']) || $attributes['REL'] == 'alternate')) { + if ($this->element == 'item') { + $this->items[$this->item]['link'] = $attributes['HREF']; + } + else { + $this->channel['link'] = $attributes['HREF']; + } + } + break; + case 'item': + $this->element = $name; + $this->item += 1; + break; + case 'entry': + $this->element = 'item'; + $this->item += 1; + break; + } + + $this->tag = $name; + } + + /** + * Performs an action when a closing tag is encountered. + */ + protected function elementEnd($parser, $name) { + switch ($name) { + case 'image': + case 'textinput': + case 'item': + case 'entry': + case 'info': + $this->element = ''; + break; + case 'id': + case 'content': + if ($this->element == $name) { + $this->element = ''; + } + } + } + + /** + * Performs an action when data is encountered. + */ + function elementData($parser, $data) { + $this->items += array($this->item => array()); + switch ($this->element) { + case 'item': + $this->items[$this->item] += array($this->tag => ''); + $this->items[$this->item][$this->tag] .= $data; + break; + case 'image': + case 'logo': + $this->image += array($this->tag => ''); + $this->image[$this->tag] .= $data; + break; + case 'link': + if ($data) { + $this->items[$this->item] += array($tag => ''); + $this->items[$this->item][$this->tag] .= $data; + } + break; + case 'content': + $this->items[$this->item] += array('content' => ''); + $this->items[$this->item]['content'] .= $data; + break; + case 'summary': + $this->items[$this->item] += array('summary' => ''); + $this->items[$this->item]['summary'] .= $data; + break; + case 'tagline': + case 'subtitle': + $this->channel += array('description' => ''); + $this->channel['description'] .= $data; + break; + case 'info': + case 'id': + case 'textinput': + // The sub-element is not supported. However, we must recognize + // it or its contents will end up in the item array. + break; + default: + $this->channel += array($this->tag => ''); + $this->channel[$this->tag] .= $data; + } + } + + /** + * Parses the W3C date/time format, a subset of ISO 8601. + * + * PHP date parsing functions do not handle this format. See + * http://www.w3.org/TR/NOTE-datetime for more information. Originally from + * MagpieRSS (http://magpierss.sourceforge.net/). + * + * @param $date_str + * A string with a potentially W3C DTF date. + * + * @return + * A timestamp if parsed successfully or FALSE if not. + */ + function parseW3cdtf($date_str) { + if (preg_match('/(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2})(:(\d{2}))?(?:([-+])(\d{2}):?(\d{2})|(Z))?/', $date_str, $match)) { + list($year, $month, $day, $hours, $minutes, $seconds) = array($match[1], $match[2], $match[3], $match[4], $match[5], $match[6]); + // Calculate the epoch for current date assuming GMT. + $epoch = gmmktime($hours, $minutes, $seconds, $month, $day, $year); + if ($match[10] != 'Z') { // Z is zulu time, aka GMT + list($tz_mod, $tz_hour, $tz_min) = array($match[8], $match[9], $match[10]); + // Zero out the variables. + if (!$tz_hour) { + $tz_hour = 0; + } + if (!$tz_min) { + $tz_min = 0; + } + $offset_secs = (($tz_hour * 60) + $tz_min) * 60; + // Is timezone ahead of GMT? If yes, subtract offset. + if ($tz_mod == '+') { + $offset_secs *= -1; + } + $epoch += $offset_secs; + } + return $epoch; + } + else { + return FALSE; + } + } +} diff --git a/core/modules/aggregator/lib/Drupal/aggregator/Plugin/aggregator/processor/DefaultProcessor.php b/core/modules/aggregator/lib/Drupal/aggregator/Plugin/aggregator/processor/DefaultProcessor.php new file mode 100644 index 0000000..a04f54c --- /dev/null +++ b/core/modules/aggregator/lib/Drupal/aggregator/Plugin/aggregator/processor/DefaultProcessor.php @@ -0,0 +1,209 @@ +get('processors'); + $info = $this->getDefinition(); + $items = drupal_map_assoc(array(3, 5, 10, 15, 20, 25), array($this, 'formatItems')); + $period = drupal_map_assoc(array(3600, 10800, 21600, 32400, 43200, 86400, 172800, 259200, 604800, 1209600, 2419200, 4838400, 9676800), 'format_interval'); + $period[AGGREGATOR_CLEAR_NEVER] = t('Never'); + + $form['processors'][$info['id']] = array(); + // Only wrap into details if there is a basic configuration. + if (isset($form['basic_conf'])) { + $form['processors'][$info['id']] = array( + '#type' => 'details', + '#title' => t('Default processor settings'), + '#description' => $info['description'], + '#collapsed' => !in_array($info['id'], $processors), + ); + } + + $form['processors'][$info['id']]['aggregator_summary_items'] = array( + '#type' => 'select', + '#title' => t('Number of items shown in listing pages'), + '#default_value' => $config->get('source.list_max'), + '#empty_value' => 0, + '#options' => $items, + ); + + $form['processors'][$info['id']]['aggregator_clear'] = array( + '#type' => 'select', + '#title' => t('Discard items older than'), + '#default_value' => $config->get('items.expire'), + '#options' => $period, + '#description' => t('Requires a correctly configured cron maintenance task.', array('@cron' => url('admin/reports/status'))), + ); + + $form['processors'][$info['id']]['aggregator_category_selector'] = array( + '#type' => 'radios', + '#title' => t('Select categories using'), + '#default_value' => $config->get('source.category_selector'), + '#options' => array('checkboxes' => t('checkboxes'), + 'select' => t('multiple selector')), + '#description' => t('For a small number of categories, checkboxes are easier to use, while a multiple selector works well with large numbers of categories.'), + ); + $form['processors'][$info['id']]['aggregator_teaser_length'] = array( + '#type' => 'select', + '#title' => t('Length of trimmed description'), + '#default_value' => $config->get('items.teaser_length'), + '#options' => drupal_map_assoc(array(0, 200, 400, 600, 800, 1000, 1200, 1400, 1600, 1800, 2000), array($this, 'formatCharacters')), + '#description' => t("The maximum number of characters used in the trimmed version of content.") + ); + return $form; + } + + /** + * Implements \Drupal\aggregator\Plugin\ProcessorInterface::settingsSubmit(). + */ + public function settingsSubmit(array $form, array &$form_state) { + $config = config('aggregator.settings'); + $config->set('items.expire', $form_state['values']['aggregator_clear']) + ->set('items.teaser_length', $form_state['values']['aggregator_teaser_length']) + ->set('source.list_max', $form_state['values']['aggregator_summary_items']) + ->set('source.category_selector', $form_state['values']['aggregator_category_selector']) + ->save(); + } + + /** + * Implements \Drupal\aggregator\Plugin\ProcessorInterface::process(). + */ + public function process(Feed $feed) { + if (!is_array($feed->items)) { + return; + } + foreach ($feed->items as $item) { + // @todo: The default entity render controller always returns an empty + // array, which is ignored in aggregator_save_item() currently. Should + // probably be fixed. + if (empty($item['title'])) { + continue; + } + + // Save this item. Try to avoid duplicate entries as much as possible. If + // we find a duplicate entry, we resolve it and pass along its ID is such + // that we can update it if needed. + if (!empty($item['guid'])) { + $values = array('fid' => $feed->id(), 'guid' => $item['guid']); + } + elseif ($item['link'] && $item['link'] != $feed->link && $item['link'] != $feed->url) { + $values = array('fid' => $feed->id(), 'link' => $item['link']); + } + else { + $values = array('fid' => $feed->id(), 'title' => $item['title']); + } + + // Try to load an existing entry. + if ($entry = entity_load_multiple_by_properties('aggregator_item', $values)) { + $entry = reset($entry); + } + else { + $entry = entity_create('aggregator_item', array('langcode' => $feed->language()->langcode)); + } + if ($item['timestamp']) { + $entry->timestamp->value = $item['timestamp']; + } + + // Make sure the item title and author fit in the 255 varchar column. + $entry->title->value = truncate_utf8($item['title'], 255, TRUE, TRUE); + $entry->author->value = truncate_utf8($item['author'], 255, TRUE, TRUE); + + $entry->fid->value = $feed->id(); + $entry->link->value = $item['link']; + $entry->description->value = $item['description']; + $entry->guid->value = $item['guid']; + $entry->save(); + } + } + + /** + * Implements \Drupal\aggregator\Plugin\ProcessorInterface::remove(). + */ + public function remove(Feed $feed) { + $iids = Database::getConnection()->query('SELECT iid FROM {aggregator_item} WHERE fid = :fid', array(':fid' => $feed->id()))->fetchCol(); + if ($iids) { + entity_delete_multiple('aggregator_item', $iids); + } + // @todo This should probably be removed. + drupal_set_message(t('The news items from %site have been removed.', array('%site' => $feed->label()))); + } + + /** + * Implements \Drupal\aggregator\Plugin\ProcessorInterface::postProcess(). + * Expires items from a feed depending on expiration settings. + */ + public function postProcess(Feed $feed) { + $aggregator_clear = config('aggregator.settings')->get('items.expire'); + + if ($aggregator_clear != AGGREGATOR_CLEAR_NEVER) { + // Remove all items that are older than flush item timer. + $age = REQUEST_TIME - $aggregator_clear; + $iids = Database::getConnection()->query('SELECT iid FROM {aggregator_item} WHERE fid = :fid AND timestamp < :timestamp', array( + ':fid' => $feed->id(), + ':timestamp' => $age, + )) + ->fetchCol(); + if ($iids) { + entity_delete_multiple('aggregator_item', $iids); + } + } + } + + /** + * Helper function for drupal_map_assoc. + * + * @param $count + * Items count. + * + * @return + * A string that is plural-formatted as "@count items". + */ + protected function formatItems($count) { + return format_plural($count, '1 item', '@count items'); + } + + /** + * Creates display text for teaser length option values. + * + * Callback for drupal_map_assoc() within settingsForm(). + * + * @param int $length + * The desired length of teaser text, in bytes. + * + * @return string + * A translated string explaining the teaser string length. + */ + protected function formatCharacters($length) { + return ($length == 0) ? t('Unlimited') : format_plural($length, '1 character', '@count characters'); + } +} diff --git a/core/modules/aggregator/lib/Drupal/aggregator/Tests/AggregatorConfigurationTest.php b/core/modules/aggregator/lib/Drupal/aggregator/Tests/AggregatorConfigurationTest.php index dbddd69..bad753e 100644 --- a/core/modules/aggregator/lib/Drupal/aggregator/Tests/AggregatorConfigurationTest.php +++ b/core/modules/aggregator/lib/Drupal/aggregator/Tests/AggregatorConfigurationTest.php @@ -23,12 +23,22 @@ public static function getInfo() { * Tests the settings form to ensure the correct default values are used. */ function testSettingsPage() { + $this->drupalGet('admin/config/services/aggregator/settings'); + // Make sure that test plugins are present. + $this->assertText(t('Test fetcher')); + $this->assertText(t('Test parser')); + $this->assertText(t('Test processor')); + + // Set new values and enable test plugins. $edit = array( 'aggregator_allowed_html_tags' => '', 'aggregator_summary_items' => 10, 'aggregator_clear' => 3600, 'aggregator_category_selector' => 'select', 'aggregator_teaser_length' => 200, + 'aggregator_fetcher' => 'aggregator_test_fetcher', + 'aggregator_parser' => 'aggregator_test_parser', + 'aggregator_processors[aggregator_test_processor]' => 'aggregator_test_processor', ); $this->drupalPost('admin/config/services/aggregator/settings', $edit, t('Save configuration')); $this->assertText(t('The configuration options have been saved.')); @@ -36,5 +46,22 @@ function testSettingsPage() { foreach ($edit as $name => $value) { $this->assertFieldByName($name, $value, format_string('"@name" has correct default value.', array('@name' => $name))); } + + // Check for our test processor settings form. + $this->assertText(t('Dummy length setting')); + // Change its value to ensure that settingsSubmit is called. + $edit = array( + 'dummy_length' => 100, + ); + $this->drupalPost('admin/config/services/aggregator/settings', $edit, t('Save configuration')); + $this->assertText(t('The configuration options have been saved.')); + $this->assertFieldByName('dummy_length', 100, '"dummy_length" has correct default value.'); + + // Make sure settings form is still accessible even after disabling a module + // that provides the selected plugins. + module_disable(array('aggregator_test')); + $this->drupalGet('admin/config/services/aggregator/settings'); + // @todo Re-enable when http://drupal.org/node/1780396 is fixed + //$this->assertResponse(200); } } diff --git a/core/modules/aggregator/lib/Drupal/aggregator/Tests/AggregatorTestBase.php b/core/modules/aggregator/lib/Drupal/aggregator/Tests/AggregatorTestBase.php index c061c3b..b43e4f8 100644 --- a/core/modules/aggregator/lib/Drupal/aggregator/Tests/AggregatorTestBase.php +++ b/core/modules/aggregator/lib/Drupal/aggregator/Tests/AggregatorTestBase.php @@ -149,9 +149,9 @@ function getDefaultFeedItemCount() { * @param \Drupal\aggregator\Plugin\Core\Entity\Feed $feed * Feed object representing the feed. * @param $expected_count - * Expected number of feed items. + * Expected number of feed items. If omitted no check will happen. */ - function updateFeedItems(Feed $feed, $expected_count) { + function updateFeedItems(Feed $feed, $expected_count = NULL) { // First, let's ensure we can get to the rss xml. $this->drupalGet($feed->url->value); $this->assertResponse(200, format_string('!url is reachable.', array('!url' => $feed->url->value))); @@ -171,8 +171,11 @@ function updateFeedItems(Feed $feed, $expected_count) { foreach ($result as $item) { $feed->items[] = $item->iid; } - $feed->item_count = count($feed->items); - $this->assertEqual($expected_count, $feed->item_count, format_string('Total items in feed equal to the total items in database (!val1 != !val2)', array('!val1' => $expected_count, '!val2' => $feed->item_count))); + + if ($expected_count !== NULL) { + $feed->item_count = count($feed->items); + $this->assertEqual($expected_count, $feed->item_count, format_string('Total items in feed equal to the total items in database (!val1 != !val2)', array('!val1' => $expected_count, '!val2' => $feed->item_count))); + } } /** @@ -361,4 +364,18 @@ function createSampleNodes($count = 5) { $this->drupalPost('node/add/article', $edit, t('Save')); } } + + /** + * Enable the plugins coming with aggregator_test module. + */ + function enableTestPlugins() { + config('aggregator.settings') + ->set('fetcher', 'aggregator_test_fetcher') + ->set('parser', 'aggregator_test_parser') + ->set('processors', array( + 'aggregator_test_processor' => 'aggregator_test_processor', + 'aggregator' => 'aggregator' + )) + ->save(); + } } diff --git a/core/modules/aggregator/lib/Drupal/aggregator/Tests/FeedFetcherPluginTest.php b/core/modules/aggregator/lib/Drupal/aggregator/Tests/FeedFetcherPluginTest.php new file mode 100644 index 0000000..6f87aef --- /dev/null +++ b/core/modules/aggregator/lib/Drupal/aggregator/Tests/FeedFetcherPluginTest.php @@ -0,0 +1,54 @@ + 'Feed fetcher plugins', + 'description' => 'Test the fetcher plugins functionality and discoverability.', + 'group' => 'Aggregator', + ); + } + + /** + * Overrides \Drupal\simpletest\WebTestBase::setUp(). + */ + public function setUp() { + parent::setUp(); + // Enable test plugins. + $this->enableTestPlugins(); + // Create some nodes. + $this->createSampleNodes(); + } + + /** + * Test fetching functionality. + */ + public function testfetch() { + // Create feed with local url. + $feed = $this->createFeed(); + $this->updateFeedItems($feed); + $this->assertFalse(empty($feed->items)); + + // Remove items and restore checked property to 0. + $this->removeFeedItems($feed); + // Change its name and try again. + $feed->title->value = 'Test feed'; + $feed->save(); + $this->updateFeedItems($feed); + // Fetch should fail due to feed name. + $this->assertTrue(empty($feed->items)); + } +} diff --git a/core/modules/aggregator/lib/Drupal/aggregator/Tests/FeedProcessorPluginTest.php b/core/modules/aggregator/lib/Drupal/aggregator/Tests/FeedProcessorPluginTest.php new file mode 100644 index 0000000..8b157d5 --- /dev/null +++ b/core/modules/aggregator/lib/Drupal/aggregator/Tests/FeedProcessorPluginTest.php @@ -0,0 +1,69 @@ + 'Feed processor plugins', + 'description' => 'Test the processor plugins functionality and discoverability.', + 'group' => 'Aggregator', + ); + } + + /** + * Overrides \Drupal\simpletest\WebTestBase::setUp(). + */ + public function setUp() { + parent::setUp(); + // Enable test plugins. + $this->enableTestPlugins(); + // Create some nodes. + $this->createSampleNodes(); + } + + /** + * Test processing functionality. + */ + public function testProcess() { + $feed = $this->createFeed(); + $this->updateFeedItems($feed); + foreach ($feed->items as $iid) { + $item = entity_load('aggregator_item', $iid); + $this->assertTrue(strpos($item->title->value, 'testProcessor') === 0); + } + } + + /** + * Test removing functionality. + */ + public function testRemove() { + $feed = $this->createFeed(); + $this->updateAndRemove($feed, NULL); + // Make sure the feed title is changed. + $entities = entity_load_multiple_by_properties('aggregator_feed', array('title' => $feed->label())); + $this->assertTrue(empty($entities)); + } + + /** + * Test post-processing functionality. + */ + public function testPostProcess() { + $feed = $this->createFeed(); + $this->updateFeedItems($feed); + // Reload the feed to get new values. + $feed = entity_load('aggregator_feed', $feed->id(), TRUE); + $this->assertEqual($feed->refresh->value, AGGREGATOR_CLEAR_NEVER); + } +} diff --git a/core/modules/aggregator/lib/Drupal/aggregator/Tests/UpdateFeedItemTest.php b/core/modules/aggregator/lib/Drupal/aggregator/Tests/UpdateFeedItemTest.php index 54eaec5..2cc1c8c 100644 --- a/core/modules/aggregator/lib/Drupal/aggregator/Tests/UpdateFeedItemTest.php +++ b/core/modules/aggregator/lib/Drupal/aggregator/Tests/UpdateFeedItemTest.php @@ -68,5 +68,12 @@ function testUpdateFeedItem() { $after = db_query('SELECT timestamp FROM {aggregator_item} WHERE fid = :fid', array(':fid' => $feed->id()))->fetchField(); $this->assertTrue($before === $after, format_string('Publish timestamp of feed item was not updated (!before === !after)', array('!before' => $before, '!after' => $after))); + + // Make sure updating items works even after disabling a module + // that provides the selected plugins. + $this->enableTestPlugins(); + module_disable(array('aggregator_test')); + $this->updateFeedItems($feed); + $this->assertResponse(200); } } diff --git a/core/modules/aggregator/tests/config/aggregator_test.settings.yml b/core/modules/aggregator/tests/config/aggregator_test.settings.yml new file mode 100644 index 0000000..f858e50 --- /dev/null +++ b/core/modules/aggregator/tests/config/aggregator_test.settings.yml @@ -0,0 +1,2 @@ +items: + dummy_length: 5 diff --git a/core/modules/aggregator/tests/lib/Drupal/aggregator_test/Plugin/aggregator/fetcher/TestFetcher.php b/core/modules/aggregator/tests/lib/Drupal/aggregator_test/Plugin/aggregator/fetcher/TestFetcher.php new file mode 100644 index 0000000..5688434 --- /dev/null +++ b/core/modules/aggregator/tests/lib/Drupal/aggregator_test/Plugin/aggregator/fetcher/TestFetcher.php @@ -0,0 +1,39 @@ +label() == 'Test feed') { + return FALSE; + } + return parent::fetch($feed); + } +} diff --git a/core/modules/aggregator/tests/lib/Drupal/aggregator_test/Plugin/aggregator/parser/TestParser.php b/core/modules/aggregator/tests/lib/Drupal/aggregator_test/Plugin/aggregator/parser/TestParser.php new file mode 100644 index 0000000..011ebc4 --- /dev/null +++ b/core/modules/aggregator/tests/lib/Drupal/aggregator_test/Plugin/aggregator/parser/TestParser.php @@ -0,0 +1,37 @@ +get('processors'); + $info = $this->getDefinition(); + + $form['processors'][$info['id']] = array( + '#type' => 'details', + '#title' => t('Test processor settings'), + '#description' => $info['description'], + '#collapsed' => !in_array($info['id'], $processors), + ); + // Add some dummy settings to verify settingsForm is called. + $form['processors'][$info['id']]['dummy_length'] = array( + '#title' => t('Dummy length setting'), + '#type' => 'number', + '#min' => 1, + '#max' => 1000, + '#default_value' => config('aggregator_test.settings')->get('items.dummy_length'), + ); + return $form; + } + + /** + * Implements \Drupal\aggregator\Plugin\ProcessorInterface::settingsSubmit(). + */ + public function settingsSubmit(array $form, array &$form_state) { + config('aggregator_test.settings') + ->set('items.dummy_length', $form_state['values']['dummy_length']) + ->save(); + } + + /** + * Implements \Drupal\aggregator\Plugin\ProcessorInterface::process(). + */ + public function process(Feed $feed) { + foreach ($feed->items as &$item) { + // Do not truncate on 255 but 240, so we prepend our test string. + $item['title'] = truncate_utf8($item['title'], 240, TRUE, TRUE); + $item['title'] = 'testProcessor' . $item['title']; + } + } + + /** + * Implements \Drupal\aggregator\Plugin\ProcessorInterface::remove(). + */ + public function remove(Feed $feed) { + parent::remove($feed); + // Remove feed title. + $feed->title->value = ''; + $feed->save(); + } + + /** + * Implements \Drupal\aggregator\Plugin\ProcessorInterface::postProcess(). + */ + public function postProcess(Feed $feed) { + // Dont let this feed be updated again. + $feed->refresh->value = AGGREGATOR_CLEAR_NEVER; + $feed->save(); + } +}