Index: modules/aggregator/aggregator-item.tpl.php =================================================================== RCS file: /cvs/drupal/drupal/modules/aggregator/aggregator-item.tpl.php,v retrieving revision 1.2 diff -p -u -r1.2 aggregator-item.tpl.php --- modules/aggregator/aggregator-item.tpl.php 15 May 2008 21:27:32 -0000 1.2 +++ modules/aggregator/aggregator-item.tpl.php 21 Jul 2008 19:47:39 -0000 @@ -1,5 +1,5 @@ - -
- : + +
+ :
Index: modules/aggregator/aggregator.admin.inc =================================================================== RCS file: /cvs/drupal/drupal/modules/aggregator/aggregator.admin.inc,v retrieving revision 1.11 diff -u -p -r1.11 aggregator.admin.inc --- modules/aggregator/aggregator.admin.inc 3 Aug 2008 05:46:55 -0000 1.11 +++ modules/aggregator/aggregator.admin.inc 5 Aug 2008 13:49:15 -0000 @@ -3,102 +3,69 @@ /** * @file - * Admin page callbacks for the aggregator module. + * Administrative form for aggregator. */ /** - * Menu callback; displays the aggregator administration page. + * Provides a content-type tab form. + * Processors and parsers should do form_alter() to provide their settings here */ -function aggregator_admin_overview() { - return aggregator_view(); -} - -/** - * Displays the aggregator administration page. - * - * @return - * The page HTML. - */ -function aggregator_view() { - $result = db_query('SELECT f.*, COUNT(i.iid) AS items FROM {aggregator_feed} f LEFT JOIN {aggregator_item} i ON f.fid = i.fid GROUP BY f.fid, f.title, f.url, f.refresh, f.checked, f.link, f.description, f.etag, f.modified, f.image, f.block ORDER BY f.title'); - - $output = '

' . t('Feed overview') . '

'; - - $header = array(t('Title'), t('Items'), t('Last update'), t('Next update'), array('data' => t('Operations'), 'colspan' => '3')); - $rows = array(); - while ($feed = db_fetch_object($result)) { - $rows[] = array(l($feed->title, "aggregator/sources/$feed->fid"), format_plural($feed->items, '1 item', '@count items'), ($feed->checked ? t('@time ago', array('@time' => format_interval(time() - $feed->checked))) : t('never')), ($feed->checked ? t('%time left', array('%time' => format_interval($feed->checked + $feed->refresh - time()))) : t('never')), l(t('edit'), "admin/content/aggregator/edit/feed/$feed->fid"), l(t('remove items'), "admin/content/aggregator/remove/$feed->fid"), l(t('update items'), "admin/content/aggregator/update/$feed->fid")); - } - $output .= theme('table', $header, $rows); - - $result = db_query('SELECT c.cid, c.title, count(ci.iid) as items FROM {aggregator_category} c LEFT JOIN {aggregator_category_item} ci ON c.cid = ci.cid GROUP BY c.cid, c.title ORDER BY title'); - - $output .= '

' . t('Category overview') . '

'; - - $header = array(t('Title'), t('Items'), t('Operations')); - $rows = array(); - while ($category = db_fetch_object($result)) { - $rows[] = array(l($category->title, "aggregator/categories/$category->cid"), format_plural($category->items, '1 item', '@count items'), l(t('edit'), "admin/content/aggregator/edit/category/$category->cid")); - } - $output .= theme('table', $header, $rows); - - return $output; -} - -/** - * Form builder; Generate a form to add/edit feed sources. - * - * @ingroup forms - * @see aggregator_form_feed_validate() - * @see aggregator_form_feed_submit() - */ -function aggregator_form_feed(&$form_state, $edit = array('refresh' => 900, 'title' => '', 'url' => '', 'fid' => NULL)) { - $period = drupal_map_assoc(array(900, 1800, 3600, 7200, 10800, 21600, 32400, 43200, 64800, 86400, 172800, 259200, 604800, 1209600, 2419200), 'format_interval'); - - if ($edit['refresh'] == '') { - $edit['refresh'] = 3600; - } - - $form['title'] = array('#type' => 'textfield', - '#title' => t('Title'), - '#default_value' => $edit['title'], - '#maxlength' => 255, - '#description' => t('The name of the feed (or the name of the website providing the feed).'), - '#required' => TRUE, - ); - $form['url'] = array('#type' => 'textfield', - '#title' => t('URL'), - '#default_value' => $edit['url'], - '#maxlength' => 255, - '#description' => t('The fully-qualified URL of the feed.'), - '#required' => TRUE, +function aggregator_settings_form($form_state) { + $type = str_replace('-', '_', arg(3)); + $form = array(); + $period = array('-1' => t('none')); + $period += drupal_map_assoc(array(900, 1800, 3600, 7200, 10800, 21600, 32400, 43200, 64800, 86400, 172800, 259200, 604800, 1209600, 2419200), 'format_interval'); + $parsers = module_implements('aggregator_parse'); + foreach ($parsers as $k => $v) { + $info = module_invoke($v, 'aggregator_parse', 'info'); + unset($parsers[$k]); + $parsers[$v] = $info['title']; + } + $processors = module_implements('aggregator_process'); + foreach ($processors as $k => $v) { + $info = module_invoke($v, 'aggregator_process', 'info'); + unset($processors[$k]); + $processors[$v] = $info['title']; + } + $form['type'] = array('#type' => 'value', '#value' => $type); // For other modules' form_alter(). + $form['aggregator_feed_' . $type] = array( + '#type' => 'checkbox', + '#title' => t('Is a feed content type'), + '#description' => t('Check if you would like to use this content type for downloading feeds to your site.'), + '#default_value' => variable_get('aggregator_feed_' . $type, FALSE), + '#weight' => -15, + ); + $form['aggregator_parser_' . $type] = array( + '#type' => 'radios', + '#title' => t('Parser'), + '#description' => t('Parsers are responsible for retrieving and parsing feed data.'), + '#options' => $parsers, + '#default_value' => variable_get('aggregator_parser_' . $type, array_pop($parsers)), + ); + $form['aggregator_processor_' . $type] = array( + '#type' => 'checkboxes', + '#title' => t('Processors'), + '#description' => t('Processors act on parsed feed data. Pick the processors suitable for your task.'), + '#options' => $processors, + '#default_value' => variable_get('aggregator_processor_' . $type, array_slice($processors, 0, 1)), + ); + $form['aggregator_deduper_' . $type] = array( + '#type' => 'hidden', + '#value' => variable_get('aggregator_deduper_' . $type, array_pop(variable_get('aggregator_processor_' . $type, array_slice($processors, 0, 1)))), ); - $form['refresh'] = array('#type' => 'select', + $form['aggregator_refresh_' . $type] = array( + '#type' => 'select', '#title' => t('Update interval'), - '#default_value' => $edit['refresh'], + '#default_value' => variable_get('aggregator_refresh_' . $type, 3600), '#options' => $period, - '#description' => t('The length of time between feed updates. (Requires a correctly configured cron maintenance task.)', array('@cron' => url('admin/reports/status'))), + '#description' => t('The approximate length of time between feed updates. (Requires a correctly configured cron maintenance task.)', array('@cron' => url('admin/reports/status'))), ); - - // Handling of categories. - $options = array(); - $values = array(); - $categories = db_query('SELECT c.cid, c.title, f.fid FROM {aggregator_category} c LEFT JOIN {aggregator_category_feed} f ON c.cid = f.cid AND f.fid = %d ORDER BY title', $edit['fid']); - while ($category = db_fetch_object($categories)) { - $options[$category->cid] = check_plain($category->title); - if ($category->fid) $values[] = $category->cid; - } - if ($options) { - $form['category'] = array( - '#type' => 'checkboxes', - '#title' => t('Categorize news items'), - '#default_value' => $values, - '#options' => $options, - '#description' => t('New feed items are automatically filed in the checked categories.'), - ); - } + $form['modules'] = array(); $form['submit'] = array( '#type' => 'submit', +<<<<<<< aggregator.admin.inc + '#value' => t('Save configuration'), +======= '#value' => t('Save'), ); @@ -395,102 +362,14 @@ function aggregator_admin_settings() { '#type' => 'select', '#title' => t('Discard items older than'), '#default_value' => variable_get('aggregator_clear', 9676800), '#options' => $period, '#description' => t('The length of time to retain feed items before discarding. (Requires a correctly configured cron maintenance task.)', array('@cron' => url('admin/reports/status'))), +>>>>>>> 1.11 ); - - $form['aggregator_category_selector'] = array( - '#type' => 'radios', '#title' => t('Category selection type'), '#default_value' => variable_get('aggregator_category_selector', 'checkboxes'), - '#options' => array('checkboxes' => t('checkboxes'), 'select' => t('multiple selector')), - '#description' => t('The type of category selection widget displayed on categorization pages. (For a small number of categories, checkboxes are easier to use, while a multiple selector work well with large numbers of categories.)'), - ); - - return system_settings_form($form); -} - -/** - * Form builder; Generate a form to add/edit/delete aggregator categories. - * - * @ingroup forms - * @see aggregator_form_category_validate() - * @see aggregator_form_category_submit() - */ -function aggregator_form_category(&$form_state, $edit = array('title' => '', 'description' => '', 'cid' => NULL)) { - $form['title'] = array('#type' => 'textfield', - '#title' => t('Title'), - '#default_value' => $edit['title'], - '#maxlength' => 64, - '#required' => TRUE, - ); - $form['description'] = array('#type' => 'textarea', - '#title' => t('Description'), - '#default_value' => $edit['description'], - ); - $form['submit'] = array('#type' => 'submit', '#value' => t('Save')); - - if ($edit['cid']) { - $form['delete'] = array('#type' => 'submit', '#value' => t('Delete')); - $form['cid'] = array('#type' => 'hidden', '#value' => $edit['cid']); - } - return $form; } /** - * Validate aggregator_form_feed form submissions. - */ -function aggregator_form_category_validate($form, &$form_state) { - if ($form_state['values']['op'] == t('Save')) { - // Check for duplicate titles - if (isset($form_state['values']['cid'])) { - $category = db_fetch_object(db_query("SELECT cid FROM {aggregator_category} WHERE title = '%s' AND cid <> %d", $form_state['values']['title'], $form_state['values']['cid'])); - } - else { - $category = db_fetch_object(db_query("SELECT cid FROM {aggregator_category} WHERE title = '%s'", $form_state['values']['title'])); - } - if ($category) { - form_set_error('title', t('A category named %category already exists. Please enter a unique title.', array('%category' => $form_state['values']['title']))); - } - } -} - -/** - * Process aggregator_form_category form submissions. - * - * @todo Add delete confirmation dialog. + * Stores the values in the {variable} table. */ -function aggregator_form_category_submit($form, &$form_state) { - if ($form_state['values']['op'] == t('Delete')) { - $title = $form_state['values']['title']; - // Unset the title. - unset($form_state['values']['title']); - } - aggregator_save_category($form_state['values']); - if (isset($form_state['values']['cid'])) { - if (isset($form_state['values']['title'])) { - drupal_set_message(t('The category %category has been updated.', array('%category' => $form_state['values']['title']))); - if (arg(0) == 'admin') { - $form_state['redirect'] = 'admin/content/aggregator/'; - return; - } - else { - $form_state['redirect'] = 'aggregator/categories/' . $form_state['values']['cid']; - return; - } - } - else { - watchdog('aggregator', 'Category %category deleted.', array('%category' => $title)); - drupal_set_message(t('The category %category has been deleted.', array('%category' => $title))); - if (arg(0) == 'admin') { - $form_state['redirect'] = 'admin/content/aggregator/'; - return; - } - else { - $form_state['redirect'] = 'aggregator/categories/'; - return; - } - } - } - else { - watchdog('aggregator', 'Category %category added.', array('%category' => $form_state['values']['title']), WATCHDOG_NOTICE, l(t('view'), 'admin/content/aggregator')); - drupal_set_message(t('The category %category has been added.', array('%category' => $form_state['values']['title']))); - } +function aggregator_settings_form_submit($form, &$form_state) { + system_settings_form_submit($form, $form_state); } Index: modules/aggregator/aggregator.info =================================================================== RCS file: /cvs/drupal/drupal/modules/aggregator/aggregator.info,v retrieving revision 1.7 diff -p -u -r1.7 aggregator.info --- modules/aggregator/aggregator.info 15 May 2008 21:27:32 -0000 1.7 +++ modules/aggregator/aggregator.info 21 Jul 2008 19:47:40 -0000 @@ -1,4 +1,4 @@ -; $Id: aggregator.info,v 1.7 2008/05/15 21:27:32 dries Exp $ +; $Id: aggregator.info,v 1.7 2008/07/21 12:15:44 aronnovak Exp $ name = Aggregator description = "Aggregates syndicated content (RSS, RDF, and Atom feeds)." @@ -8,3 +8,4 @@ core = 7.x files[] = aggregator.module files[] = aggregator.admin.inc files[] = aggregator.pages.inc +files[] = aggregator.light.inc Index: modules/aggregator/aggregator.install =================================================================== RCS file: /cvs/drupal/drupal/modules/aggregator/aggregator.install,v retrieving revision 1.16 diff -p -u -r1.16 aggregator.install --- modules/aggregator/aggregator.install 15 May 2008 21:27:32 -0000 1.16 +++ modules/aggregator/aggregator.install 21 Jul 2008 19:47:44 -0000 @@ -1,5 +1,5 @@ $type_name, + 'name' => st('Feed'), + 'module' => 'node', + 'description' => st("A feed is for aggregating syndicated content from other sites."), + 'custom' => TRUE, + 'modified' => TRUE, + 'locked' => FALSE, + 'help' => '', + 'min_word_count' => '', + ); + $type = (object) _node_type_set_defaults($type); + node_type_save($type); + variable_set('aggregator_parser_' . $type_name, 'syndication_parser'); + variable_set('aggregator_processor_' . $type_name, unserialize('a:1:{s:10:"aggregator";s:10:"aggregator";}')); + variable_set('aggregator_feed_' . $type_name, 1); + } } /** * Implementation of hook_schema(). */ function aggregator_schema() { - $schema['aggregator_category'] = array( - 'description' => t('Stores categories for aggregator feeds and feed items.'), - 'fields' => array( - 'cid' => array( - 'type' => 'serial', - 'not null' => TRUE, - 'description' => t('Primary Key: Unique aggregator category ID.'), - ), - 'title' => array( - 'type' => 'varchar', - 'length' => 255, - 'not null' => TRUE, - 'default' => '', - 'description' => t('Title of the category.'), - ), - 'description' => array( - 'type' => 'text', - 'not null' => TRUE, - 'size' => 'big', - 'description' => t('Description of the category'), - ), - 'block' => array( - 'type' => 'int', - 'not null' => TRUE, - 'default' => 0, - 'size' => 'tiny', - 'description' => t('The number of recent items to show within the category block.'), - ) - ), - 'primary key' => array('cid'), - 'unique keys' => array( - 'title' => array('title'), - ), - ); - - $schema['aggregator_category_feed'] = array( - 'description' => t('Bridge table; maps feeds to categories.'), - 'fields' => array( - 'fid' => array( - 'type' => 'int', - 'not null' => TRUE, - 'default' => 0, - 'description' => t("The feed's {aggregator_feed}.fid."), - ), - 'cid' => array( - 'type' => 'int', - 'not null' => TRUE, - 'default' => 0, - 'description' => t('The {aggregator_category}.cid to which the feed is being assigned.'), - ) - ), - 'primary key' => array('cid', 'fid'), - 'indexes' => array( - 'fid' => array('fid'), - ), - ); - - $schema['aggregator_category_item'] = array( - 'description' => t('Bridge table; maps feed items to categories.'), - 'fields' => array( - 'iid' => array( - 'type' => 'int', - 'not null' => TRUE, - 'default' => 0, - 'description' => t("The feed item's {aggregator_item}.iid."), - ), - 'cid' => array( - 'type' => 'int', - 'not null' => TRUE, - 'default' => 0, - 'description' => t('The {aggregator_category}.cid to which the feed item is being assigned.'), - ) - ), - 'primary key' => array('cid', 'iid'), - 'indexes' => array( - 'iid' => array('iid'), - ), - ); - $schema['aggregator_feed'] = array( 'description' => t('Stores feeds to be parsed by the aggregator.'), 'fields' => array( - 'fid' => array( - 'type' => 'serial', - 'not null' => TRUE, - 'description' => t('Primary Key: Unique feed ID.'), - ), - 'title' => array( - 'type' => 'varchar', - 'length' => 255, - 'not null' => TRUE, - 'default' => '', - 'description' => t('Title of the feed.'), + 'nid' => array( + 'description' => t('The {node}.nid to which this feed belongs.'), + 'type' => 'int', + 'unsigned' => TRUE, + 'not null' => TRUE ), 'url' => array( - 'type' => 'varchar', - 'length' => 255, - 'not null' => TRUE, - 'default' => '', - 'description' => t('URL to the feed.'), - ), - 'refresh' => array( - 'type' => 'int', + 'type' => 'text', 'not null' => TRUE, - 'default' => 0, - 'description' => t('How often to check for new feed items, in seconds.'), + 'description' => t('URL to the feed.') ), 'checked' => array( 'type' => 'int', @@ -140,17 +72,8 @@ function aggregator_schema() { 'description' => t('Last time feed was checked for new items, as Unix timestamp.'), ), 'link' => array( - 'type' => 'varchar', - 'length' => 255, - 'not null' => TRUE, - 'default' => '', - 'description' => t('The parent website of the feed; comes from the element in the feed.'), - ), - 'description' => array( 'type' => 'text', - 'not null' => TRUE, - 'size' => 'big', - 'description' => t("The parent website's description; comes from the element in the feed."), + 'description' => t('The parent website of the feed.'), ), 'image' => array( 'type' => 'text', @@ -171,21 +94,16 @@ function aggregator_schema() { 'default' => 0, 'description' => t('When the feed was last modified, as a Unix timestamp.'), ), - 'block' => array( - 'type' => 'int', - 'not null' => TRUE, - 'default' => 0, - 'size' => 'tiny', - 'description' => t("Number of items to display in the feed's block."), - ) + 'hash' => array( + 'description' => t('Stores the hash of the parsed feed structure.'), + 'type' => 'varchar', + 'length' => '32'), ), - 'primary key' => array('fid'), + 'primary key' => array('nid'), 'unique keys' => array( - 'url' => array('url'), - 'title' => array('title'), + 'url' => array(array('url', 100)), ), ); - $schema['aggregator_item'] = array( 'description' => t('Stores the individual items imported from feeds.'), 'fields' => array( @@ -194,11 +112,11 @@ function aggregator_schema() { 'not null' => TRUE, 'description' => t('Primary Key: Unique ID for feed item.'), ), - 'fid' => array( + 'nid' => array( 'type' => 'int', 'not null' => TRUE, 'default' => 0, - 'description' => t('The {aggregator_feed}.fid to which this item belongs.'), + 'description' => t('The {node}.nid to which this item belongs.'), ), 'title' => array( 'type' => 'varchar', @@ -230,18 +148,18 @@ function aggregator_schema() { 'timestamp' => array( 'type' => 'int', 'not null' => FALSE, - 'description' => t('Posted date of the feed item, as a Unix timestamp.'), + 'description' => t('Post date of feed item, as a Unix timestamp.'), ), 'guid' => array( 'type' => 'varchar', 'length' => 255, 'not null' => FALSE, 'description' => t('Unique identifier for the feed item.'), - ) + ), ), 'primary key' => array('iid'), 'indexes' => array( - 'fid' => array('fid'), + 'nid' => array('nid') ), ); Index: modules/aggregator/aggregator.light.inc =================================================================== RCS file: modules/aggregator/aggregator.light.inc diff -N modules/aggregator/aggregator.light.inc --- /dev/null 1 Jan 1970 00:00:00 -0000 +++ modules/aggregator/aggregator.light.inc 21 Jul 2008 19:47:44 -0000 @@ -0,0 +1,184 @@ + $name) { + $block['aggregator-' . $tid]['info'] = t('Latest items with !term term', array('!term' => $name)); + } + return $block; + } + else if ($op == 'configure') { + $value = variable_get($delta . '-block', 5); + $form['block'] = array('#type' => 'select', '#title' => t('Number of news items in block'), '#default_value' => $value, '#options' => drupal_map_assoc(array(2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20))); + return $form; + } + else if ($op == 'save') { + variable_set($delta . '-block', $edit['block']); + } + else if ($op == 'view') { + if (user_access('access content')) { + $tid = array_pop(explode('-', $delta)); + $query_args = array(); + if ($tid == 'latest') { + $title = t('Latest items'); + $query = "SELECT title, link FROM {aggregator_item} ORDER BY timestamp DESC"; + } + else if(is_numeric($tid)) { + $term = taxonomy_get_term($tid); + $title = t('Latest items with !term term', array('!term' => $term->name)); + $query = "SELECT i.title, i.link FROM {aggregator_item} i LEFT JOIN {term_node} t ON t.nid = i.nid WHERE t.tid = %d ORDER BY timestamp DESC"; + $query_args = array($tid); + } + $result = db_query_range($query, $query_args, 0, variable_get($delta . '-block', 5)); + $items = array(); + while ($item = db_fetch_object($result)) { + $items[] = theme('aggregator_block_item', $item); + } + $block['subject'] = check_markup($title, 4); + $block['content'] = theme('item_list', $items); + return $block; + } + } +} + +/** + * Implementation of hook_aggregator_process(). + * + * @param $op + * 'save' The feed items should be updated or saved. + * 'info' Metadata about the processor + * @param $node + * The feed-node object. + */ +function aggregator_aggregator_process($op, $node = NULL) { + switch ($op) { + case 'save': + if (is_array($node->feed->items)) { + $age = variable_get('aggregator_clear_' . $node->type, 9676800); + $new_items = 0; + foreach ($node->feed->items as $item) { + // Avoid to create already expired items + if ($item->unique === TRUE) { + if (time() - $item->timestamp < $age) { + db_query("INSERT INTO {aggregator_item} (nid, title, link, author, description, timestamp, guid) VALUES (%d, '%s', '%s', '%s', '%s', %d, '%s')", $node->nid, $item->title, $item->link, '', $item->description, $item->timestamp, $item->guid); + $new_items++; + } + } + else { + db_query("UPDATE {aggregator_item} SET title = '%s', author = '%s', description = '%s', timestamp = %d, guid = '%s' WHERE iid = '%s'", $item->title, '', $item->description, $item->timestamp, $item->guid, $item->unique); + } + } + return $new_items; + } + break; + case 'unique': + foreach ($node->feed->items as $k => $item) { + if (!empty($item->guid) && strlen($item->guid) < 256) { + $iid = db_result(db_query("SELECT iid FROM {aggregator_item} WHERE nid = %d AND guid = '%s'", $node->nid, $item->guid)); + } + else if ($item->link && $item->link != $node->feed->link && $item->link != $node->feed->link) { + $iid = db_result(db_query("SELECT iid FROM {aggregator_item} WHERE nid = %d AND link = '%s'", $node->nid, $item->link)); + } + else { + $iid = db_result(db_query("SELECT iid FROM {aggregator_item} WHERE nid = %d AND title = '%s'", $node->nid, $item->title)); + } + $node->feed->items[$k]->unique = ($iid == FALSE ? TRUE : $iid); + } + break; + case 'info': + return array( + 'title' => t('Aggregator Light'), + 'description' => t('Provides lightweight item processor for Aggregator.'), + ); + } +} + +/** + * Per-content-type settings for Aggregator Light processor. + */ +function _aggregator_light_form_settings(&$form) { + $type = $form['type']['#value']; + $info = module_invoke('aggregator', 'aggregator_process', 'info'); + $form['modules']['aggregator'] = array( + '#type' => 'fieldset', + '#title' => t('Aggregator Light processor settings'), + '#description' => $info['description'], + '#collapsible' => TRUE, + '#collapsed' => !aggregator_is_enabled('aggregator', $type), + ); + $vocabs = taxonomy_get_vocabularies(); + $vids = array(); + $vids[0] = t('None'); + foreach ($vocabs as $vid => $vocab) { + $vids[$vid] = $vocab->name; + } + $period = drupal_map_assoc(array(3600, 10800, 21600, 32400, 43200, 86400, 172800, 259200, 604800, 1209600, 2419200, 4838400, 9676800), 'format_interval'); + $form['modules']['aggregator']['aggregator_vid_' . $type] = array( + '#type' => 'select', + '#title' => t('Vocabulary'), + '#multiple' => FALSE, + '#options' => $vids, + '#description' => t('Select a vocabulary for categorizing feed items.'), + '#default_value' => variable_get('aggregator_vid_' . $type, 0), + ); + $form['modules']['aggregator']['aggregator_clear_' . $type] = array( + '#type' => 'select', + '#title' => t('Discard items older than'), + '#default_value' => variable_get('aggregator_clear_' . $type, 9676800), + '#options' => $period, + '#description' => t('The length of time to retain feed items before discarding. (Requires a correctly configured cron maintenance task.)', array('@cron' => url('admin/reports/status'))), + ); + $formats = filter_formats(); + foreach ($formats as $k => $format) { + $formats[$k] = $format->name; + } + $form['modules']['aggregator']['aggregator_input_filter_' . $type] = array( + '#type' => 'select', + '#title' => t('Input format for item description'), + '#default_value' => variable_get('aggregator_input_filter_' . $type, FILTER_FORMAT_DEFAULT), + '#options' => $formats, + '#description' => t('You can specify here the allowed elements in the feed items.'), + ); + $items = array(0 => t('none')) + drupal_map_assoc(array(3, 5, 10, 15, 20, 25), '_aggregator_items'); + $form['modules']['aggregator']['aggregator_summary_items_' . $type] = array( + '#type' => 'select', + '#title' => t('Items shown in sources and categories pages') , + '#default_value' => variable_get('aggregator_summary_items_' . $type, 3), + '#options' => $items, + '#description' => t('Number of feed items displayed in feed and category summary pages.'), + ); +} + +/** + * Delete expired items. + */ +function _aggregator_light_delete_expired() { + $types = node_get_types(); + // Delete expired light items. + foreach ($types as $type) { + if (variable_get('aggregator_feed_' . $type->type, FALSE) && aggregator_is_enabled('aggregator', $type->type)) { + $result = db_query("SELECT l.iid FROM {aggregator_item} l LEFT JOIN {node} n ON l.nid = n.nid WHERE n.type = '%s'", $type->type); + $iids = array(); + while ($item = db_fetch_array($result)) { + $iids[] = $item['iid']; + } + if (count($iids) > 0) { + $age = variable_get('aggregator_clear_' . $type->type, 9676800); + db_query('DELETE FROM {aggregator_item} WHERE iid IN (' . implode(', ', $iids) . ') AND %d - timestamp > %d', time(), $age); + } + } + } +} Index: modules/aggregator/aggregator.module =================================================================== RCS file: /cvs/drupal/drupal/modules/aggregator/aggregator.module,v retrieving revision 1.384 diff -u -p -r1.384 modules/aggregator/aggregator.module --- modules/aggregator/aggregator.module 3 Aug 2008 18:17:37 -0000 1.384 +++ modules/aggregator/aggregator.module 5 Aug 2008 13:49:16 -0000 @@ -16,16 +16,9 @@ function aggregator_help($path, $arg) { $output .= '

' . t('Feeds contain feed items, or individual posts published by the site providing the feed. Feeds may be grouped in categories, generally by topic. Users view feed items in the main aggregator display or by their source. Administrators can add, edit and delete feeds and choose how often to check each feed for newly updated items. The most recent items in either a feed or category can be displayed as a block through the blocks administration page. A machine-readable OPML file of all feeds is available. A correctly configured cron maintenance task is required to update feeds automatically.', array('@aggregator' => url('aggregator'), '@aggregator-sources' => url('aggregator/sources'), '@feededit' => url('admin/content/aggregator'), '@admin-block' => url('admin/build/block'), '@aggregator-opml' => url('aggregator/opml'), '@cron' => url('admin/reports/status'))) . '

'; $output .= '

' . t('For more information, see the online handbook entry for Aggregator module.', array('@aggregator' => 'http://drupal.org/handbook/modules/aggregator/')) . '

'; return $output; - case 'admin/content/aggregator': - $output = '

' . t('Thousands of sites (particularly news sites and blogs) publish their latest headlines and posts in feeds, using a number of standardized XML-based formats. Formats supported by the aggregator include RSS, RDF, and Atom.', array('@rss' => 'http://cyber.law.harvard.edu/rss/', '@rdf' => 'http://www.w3.org/RDF/', '@atom' => 'http://www.atomenabled.org')) . '

'; - $output .= '

' . t('Current feeds are listed below, and new feeds may be added. For each feed or feed category, the latest items block may be enabled at the blocks administration page.', array('@addfeed' => url('admin/content/aggregator/add/feed'), '@block' => url('admin/build/block'))) . '

'; + case 'aggregator/terms': + $output = '

' . t('You can assign terms to a feed at node edit form. Aggregator uses a selected vocabulary for each content-type. It\'s possible to alter the selected content-type at the content-type edit page Feed Aggregator tab. The feed items gets the same taxonomy information like the feed nodes. For example if the feed node is assigned with term foo, all the feed items belong to that feed will appear under foo term page.') . '

'; return $output; - case 'admin/content/aggregator/add/feed': - return '

' . t('Add a feed in RSS, RDF or Atom format. A feed may only have one entry.') . '

'; - case 'admin/content/aggregator/add/category': - return '

' . t('Categories allow feed items from different feeds to be grouped together. For example, several sport-related feeds may belong to a category named Sports. Feed items may be grouped automatically (by selecting a category when creating or editing a feed) or manually (via the Categorize page available from feed item listings). Each category provides its own feed page and block.') . '

'; - case 'admin/content/aggregator/add/opml': - return '

' . t('OPML is an XML format used to exchange multiple feeds between aggregators. A single OPML document may contain a collection of many feeds. Drupal can parse such a file and import all feeds at once, saving you the effort of adding them manually. You may either upload a local file from your computer or enter a URL where Drupal can download it.') . '

'; } } @@ -81,26 +74,25 @@ function aggregator_theme() { * Implementation of hook_menu(). */ function aggregator_menu() { - $items['admin/content/aggregator'] = array( - 'title' => 'Feed aggregator', - 'description' => "Configure which content your site aggregates from other sites, how often it polls them, and how they're categorized.", - 'page callback' => 'aggregator_admin_overview', - 'access arguments' => array('administer news feeds'), - ); - $items['admin/content/aggregator/add/feed'] = array( - 'title' => 'Add feed', - 'page callback' => 'drupal_get_form', - 'page arguments' => array('aggregator_form_feed'), - 'access arguments' => array('administer news feeds'), - 'type' => MENU_LOCAL_TASK, - 'parent' => 'admin/content/aggregator', - ); - $items['admin/content/aggregator/add/category'] = array( - 'title' => 'Add category', - 'page callback' => 'drupal_get_form', - 'page arguments' => array('aggregator_form_category'), - 'access arguments' => array('administer news feeds'), + $types = node_get_types('types', NULL, TRUE); + foreach ($types as $type) { + $in_url = str_replace('_', '-', $type->type); + $items["admin/build/node-type/$in_url/aggregator"] = array( + 'title' => 'Feed aggregator', + 'page callback' => 'drupal_get_form', + 'page arguments' => array('aggregator_settings_form', $type->type), + 'access arguments' => array('administer news feeds'), + 'type' => MENU_LOCAL_TASK, + 'parent' => "admin/build/node-type/$type->type", + ); + } + $items['node/%node/refresh'] = array( + 'title' => 'Refresh', + 'page callback' => 'aggregator_refresh_page', + 'page arguments' => array(1), 'type' => MENU_LOCAL_TASK, + 'access callback' => '_aggregator_perm_refresh', + 'access arguments' => array(1), 'parent' => 'admin/content/aggregator', ); $items['admin/content/aggregator/add/opml'] = array( @@ -141,18 +133,30 @@ function aggregator_menu() { $items['aggregator'] = array( 'title' => 'Feed aggregator', 'page callback' => 'aggregator_page_last', - 'access arguments' => array('access news feeds'), + 'access arguments' => array('access content'), 'weight' => 5, ); $items['aggregator/sources'] = array( 'title' => 'Sources', 'page callback' => 'aggregator_page_sources', - 'access arguments' => array('access news feeds'), + 'access arguments' => array('access content'), + ); + $items['aggregator/sources/%node'] = array( + 'page callback' => 'aggregator_page_source', + 'page arguments' => array(2), + 'access arguments' => array('access content'), + 'type' => MENU_CALLBACK, ); - $items['aggregator/categories'] = array( - 'title' => 'Categories', - 'page callback' => 'aggregator_page_categories', - 'access callback' => '_aggregator_has_categories', + $items['aggregator/terms'] = array( + 'title' => 'Terms', + 'page callback' => 'aggregator_page_terms', + 'access arguments' => array('access content'), + ); + $items['aggregator/terms/%'] = array( + 'title' => 'Terms', + 'page callback' => 'aggregator_page_terms', + 'page arguments' => array(2), + 'access arguments' => array('access content'), ); $items['aggregator/rss'] = array( 'title' => 'RSS feed', @@ -166,86 +170,48 @@ function aggregator_menu() { 'access arguments' => array('access news feeds'), 'type' => MENU_CALLBACK, ); - $items['aggregator/categories/%aggregator_category'] = array( - 'title callback' => '_aggregator_category_title', - 'title arguments' => array(2), - 'page callback' => 'aggregator_page_category', - 'page arguments' => array(2), - 'access callback' => 'user_access', - 'access arguments' => array('access news feeds'), - ); - $items['aggregator/categories/%aggregator_category/view'] = array( - 'title' => 'View', - 'type' => MENU_DEFAULT_LOCAL_TASK, - 'weight' => -10, - ); - $items['aggregator/categories/%aggregator_category/categorize'] = array( - 'title' => 'Categorize', - 'page callback' => 'drupal_get_form', - 'page arguments' => array('aggregator_page_category', 2), - 'access arguments' => array('administer news feeds'), - 'type' => MENU_LOCAL_TASK, - ); - $items['aggregator/categories/%aggregator_category/configure'] = array( - 'title' => 'Configure', - 'page callback' => 'drupal_get_form', - 'page arguments' => array('aggregator_form_category', 2), - 'access arguments' => array('administer news feeds'), - 'type' => MENU_LOCAL_TASK, - 'weight' => 1, - ); - $items['aggregator/sources/%aggregator_feed'] = array( - 'page callback' => 'aggregator_page_source', - 'page arguments' => array(2), - 'access arguments' => array('access news feeds'), - 'type' => MENU_CALLBACK, - ); - $items['aggregator/sources/%aggregator_feed/view'] = array( - 'title' => 'View', - 'type' => MENU_DEFAULT_LOCAL_TASK, - 'weight' => -10, - ); - $items['aggregator/sources/%aggregator_feed/categorize'] = array( - 'title' => 'Categorize', - 'page callback' => 'drupal_get_form', - 'page arguments' => array('aggregator_page_source', 2), - 'access arguments' => array('administer news feeds'), - 'type' => MENU_LOCAL_TASK, - ); - $items['aggregator/sources/%aggregator_feed/configure'] = array( - 'title' => 'Configure', - 'page callback' => 'drupal_get_form', - 'page arguments' => array('aggregator_form_feed', 2), - 'access arguments' => array('administer news feeds'), - 'type' => MENU_LOCAL_TASK, - 'weight' => 1, - ); - $items['admin/content/aggregator/edit/feed/%aggregator_feed'] = array( - 'title' => 'Edit feed', - 'page callback' => 'drupal_get_form', - 'page arguments' => array('aggregator_form_feed', 5), - 'access arguments' => array('administer news feeds'), - 'type' => MENU_CALLBACK, - ); - $items['admin/content/aggregator/edit/category/%aggregator_category'] = array( - 'title' => 'Edit category', - 'page callback' => 'drupal_get_form', - 'page arguments' => array('aggregator_form_category', 5), - 'access arguments' => array('administer news feeds'), - 'type' => MENU_CALLBACK, - ); return $items; } /** - * Menu callback. - * - * @return - * An aggregator category title. + * Implementation of hook_nodeapi(). */ -function _aggregator_category_title($category) { - return $category['title']; +function aggregator_nodeapi(&$node, $op, $teaser, $page) { + if (isset($node->feed) || variable_get('aggregator_feed_' . $node->type, FALSE)) { + switch ($op) { + case 'presave': + $node = aggregator_feed_create($node->url, $node->type, $node); + $nodes_list = array($node); + aggregator_feed_retrieve($nodes_list); + $node = $nodes_list[0]; + if (empty($node->title)) { + // The title could not be empty, so in emergency case, use the url. + $node->title = !empty($node->feed->title) ? $node->feed->title : $node->feed->url; + } + if (empty($node->body)) { + $node->body = isset($node->feed->description) ? $node->feed->description : ''; + } + break; + case 'insert': + case 'update': + aggregator_feed_save($node); + break; + case 'load': + $node->feed = db_fetch_object(db_query("SELECT * FROM {aggregator_feed} WHERE nid = %d", $node->nid)); + break; + case 'delete': + db_query("DELETE FROM {aggregator_feed} WHERE nid = %d", $node->nid); + break; + } + } + if (aggregator_is_enabled('aggregator', $node->type)) { + switch ($op) { + case 'delete': + db_query('DELETE FROM {aggregator_item} WHERE nid = %d', $node->nid); + break; + } + } } /** @@ -256,660 +222,367 @@ function aggregator_init() { } /** - * Find out whether there are any aggregator categories. - * - * @return - * TRUE if there is at least one category and the user has access to them, FALSE otherwise. - */ -function _aggregator_has_categories() { - return user_access('access news feeds') && db_result(db_query('SELECT COUNT(*) FROM {aggregator_category}')); -} - -/** - * Implementation of hook_perm(). - */ -function aggregator_perm() { - return array( - 'administer news feeds' => t('Add, edit or delete news feeds that are aggregated to your site.'), - 'access news feeds' => t('View aggregated news feed items.'), - ); -} - -/** * Implementation of hook_cron(). * * Checks news feeds for updates once their refresh interval has elapsed. + * Deletes expired items. */ function aggregator_cron() { - $result = db_query('SELECT * FROM {aggregator_feed} WHERE checked + refresh < %d', time()); - while ($feed = db_fetch_array($result)) { - aggregator_refresh($feed); + $ready = FALSE; + if (drupal_function_exists('_aggregator_light_delete_expired')) { + _aggregator_light_delete_expired(); + } + $types = node_get_types(); + $nids_not_refresh = array(); + $types_skip = array(); + // Collect the feeds where the refresh interval has not elapsed. + $start = time(); + foreach ($types as $type) { + $refresh = variable_get('aggregator_refresh_' . $type->type, 3600); + if ($refresh == -1) { + $types_skip[] = "'" . $type->type . "'"; + continue; + } + $result = db_query("SELECT f.nid FROM {aggregator_feed} f LEFT JOIN {node} n ON f.nid = n.nid WHERE n.type = '%s' AND (%d - f.checked) < %d", $type->type, $start, $refresh); + while ($nid = db_fetch_array($result)) { + $nids_not_refresh[] = $nid['nid']; + } + } + // Query the feeds which should be refreshed and do the refresh. + while(_aggregator_cron_time() || !$ready) { + $sql = "SELECT f.nid FROM {aggregator_feed} f LEFT JOIN {node} n ON n.nid = f.nid WHERE f.checked <= %d"; + $sql .= count($nids_not_refresh) > 0 ? " AND f.nid NOT IN (" . implode(', ', $nids_not_refresh) . ")" : ''; + $sql .= count($types_skip) > 0 ? " AND n.type NOT IN (" . implode(', ', $types_skip) . ")" : ''; + $result = db_query_range($sql . " ORDER BY f.checked", $start, 0, 2); + $nids = array(); + while ($nid = db_fetch_array($result)) { + $nids[] = $nid['nid']; + } + if (count($nids) == 0) { + $ready = TRUE; + } + else { + aggregator_feed_refresh($nids); + } } } /** - * Implementation of hook_block(). - * - * Generates blocks for the latest news items in each category and feed. + * Implementation of hook_form_alter(). */ -function aggregator_block($op = 'list', $delta = '', $edit = array()) { - if (user_access('access news feeds')) { - if ($op == 'list') { - $result = db_query('SELECT cid, title FROM {aggregator_category} ORDER BY title'); - while ($category = db_fetch_object($result)) { - $block['category-' . $category->cid]['info'] = t('!title category latest items', array('!title' => $category->title)); - } - $result = db_query('SELECT fid, title FROM {aggregator_feed} ORDER BY fid'); - while ($feed = db_fetch_object($result)) { - $block['feed-' . $feed->fid]['info'] = t('!title feed latest items', array('!title' => $feed->title)); - } - } - elseif ($op == 'configure') { - list($type, $id) = explode('-', $delta); - if ($type == 'category') { - $value = db_result(db_query('SELECT block FROM {aggregator_category} WHERE cid = %d', $id)); - } - else { - $value = db_result(db_query('SELECT block FROM {aggregator_feed} WHERE fid = %d', $id)); - } - $form['block'] = array('#type' => 'select', '#title' => t('Number of news items in block'), '#default_value' => $value, '#options' => drupal_map_assoc(array(2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20))); - return $form; +function aggregator_form_alter(&$form, $form_state, $form_id) { + // Alters node form in the case of aggregator-enabled content-types. + if (isset($form['type']) && isset($form['#node']) && $form['type']['#value'] .'_node_form' == $form_id && variable_get('aggregator_feed_' . $form['type']['#value'], FALSE)) { + $form['title']['#required'] = FALSE; + $url = ''; + if (isset($form_state['values']) && $form_state['values']['url']) { + $url = $form_state['values']['url']; } - elseif ($op == 'save') { - list($type, $id) = explode('-', $delta); - if ($type == 'category') { - $value = db_query('UPDATE {aggregator_category} SET block = %d WHERE cid = %d', $edit['block'], $id); - } - else { - $value = db_query('UPDATE {aggregator_feed} SET block = %d WHERE fid = %d', $edit['block'], $id); - } - } - elseif ($op == 'view') { - list($type, $id) = explode('-', $delta); - switch ($type) { - case 'feed': - if ($feed = db_fetch_object(db_query('SELECT fid, title, block FROM {aggregator_feed} WHERE fid = %d', $id))) { - $block['subject'] = check_plain($feed->title); - $result = db_query_range('SELECT * FROM {aggregator_item} WHERE fid = %d ORDER BY timestamp DESC, iid DESC', $feed->fid, 0, $feed->block); - $read_more = theme('more_link', url('aggregator/sources/' . $feed->fid), t("View this feed's recent news.")); - } - break; - - case 'category': - if ($category = db_fetch_object(db_query('SELECT cid, title, block FROM {aggregator_category} WHERE cid = %d', $id))) { - $block['subject'] = check_plain($category->title); - $result = db_query_range('SELECT i.* FROM {aggregator_category_item} ci LEFT JOIN {aggregator_item} i ON ci.iid = i.iid WHERE ci.cid = %d ORDER BY i.timestamp DESC, i.iid DESC', $category->cid, 0, $category->block); - $read_more = theme('more_link', url('aggregator/categories/' . $category->cid), t("View this category's recent news.")); - } - break; - } - $items = array(); - while ($item = db_fetch_object($result)) { - $items[] = theme('aggregator_block_item', $item); - } - - // Only display the block if there are items to show. - if (count($items) > 0) { - $block['content'] = theme('item_list', $items) . $read_more; - } - } - if (isset($block)) { - return $block; + else if (isset($form['#node']->feed->url)) { + $url = $form['#node']->feed->url; } + $form['url'] = array( + '#type' => 'textfield', + '#title' => t('Feed URL'), + '#description' => t('Enter the URL of the feed or the URL of the website where the feed can be found.'), + '#default_value' => $url, + '#maxlength' => 2048, + '#weight' => -10, + '#required' => TRUE, + ); + } + // Content-type tab form, add Aggregator Light processor specific settings. + if ($form_id == 'aggregator_settings_form' && drupal_function_exists('_aggregator_light_form_settings')) { + _aggregator_light_form_settings($form); + return $form; } } /** - * Add/edit/delete aggregator categories. - * - * @param $edit - * An associative array describing the category to be added/edited/deleted. + * Implementation of hook_requirements(). */ -function aggregator_save_category($edit) { - $link_path = 'aggregator/categories/'; - if (!empty($edit['cid'])) { - $link_path .= $edit['cid']; - if (!empty($edit['title'])) { - db_query("UPDATE {aggregator_category} SET title = '%s', description = '%s' WHERE cid = %d", $edit['title'], $edit['description'], $edit['cid']); - $op = 'update'; - } - else { - db_query('DELETE FROM {aggregator_category} WHERE cid = %d', $edit['cid']); - $edit['title'] = ''; - $op = 'delete'; - } +function aggregator_requirements($phase) { + $t = get_t(); + $requirements['aggregagor'] = array( + 'title' => t('Aggregator'), + ); + $modules = module_implements('aggregator_parse'); + if (count($modules)) { + $requirements['aggregagor']['value'] = $t('Parser module(s) installed.'); + $requirements['aggregagor']['severity'] = REQUIREMENT_OK; } - elseif (!empty($edit['title'])) { - // A single unique id for bundles and feeds, to use in blocks. - db_query("INSERT INTO {aggregator_category} (title, description, block) VALUES ('%s', '%s', 5)", $edit['title'], $edit['description']); - $link_path .= db_last_insert_id('aggregator_category', 'cid'); - $op = 'insert'; - } - if (isset($op)) { - menu_link_maintain('aggregator', $op, $link_path, $edit['title']); + else { + $requirements['aggregagor']['value'] = $t('!enable_link (e. g. Syndication Parser) for using aggregator.', array('!enable_link' => l($t('Enable at least one parser module'), 'admin/build/modules'))); + $requirements['aggregagor']['severity'] = REQUIREMENT_ERROR; } + return $requirements; } /** - * Add/edit/delete an aggregator feed. + * Downloads the given feeds, if changed, call the parsers, call the processors. * - * @param $edit - * An associative array describing the feed to be added/edited/deleted. + * @param $feeds + * Array of nid's of the feeds or array of node objects. */ -function aggregator_save_feed($edit) { - if (!empty($edit['fid'])) { - // An existing feed is being modified, delete the category listings. - db_query('DELETE FROM {aggregator_category_feed} WHERE fid = %d', $edit['fid']); - } - if (!empty($edit['fid']) && !empty($edit['title'])) { - db_query("UPDATE {aggregator_feed} SET title = '%s', url = '%s', refresh = %d WHERE fid = %d", $edit['title'], $edit['url'], $edit['refresh'], $edit['fid']); - } - elseif (!empty($edit['fid'])) { - $items = array(); - $result = db_query('SELECT iid FROM {aggregator_item} WHERE fid = %d', $edit['fid']); - while ($item = db_fetch_object($result)) { - $items[] = "iid = $item->iid"; - } - if (!empty($items)) { - db_query('DELETE FROM {aggregator_category_item} WHERE ' . implode(' OR ', $items)); - } - db_query('DELETE FROM {aggregator_feed} WHERE fid = %d', $edit['fid']); - db_query('DELETE FROM {aggregator_item} WHERE fid = %d', $edit['fid']); - } - elseif (!empty($edit['title'])) { - db_query("INSERT INTO {aggregator_feed} (title, url, refresh, block, description, image) VALUES ('%s', '%s', %d, 5, '', '')", $edit['title'], $edit['url'], $edit['refresh']); - // A single unique ID for bundles and feeds, to use in blocks. - $edit['fid'] = db_last_insert_id('aggregator_feed', 'fid'); - } - if (!empty($edit['title'])) { - // The feed is being saved, save the categories as well. - if (!empty($edit['category'])) { - foreach ($edit['category'] as $cid => $value) { - if ($value) { - db_query('INSERT INTO {aggregator_category_feed} (fid, cid) VALUES (%d, %d)', $edit['fid'], $cid); - } - } +function aggregator_feed_refresh($feeds) { + $feed_nodes = array(); + foreach ($feeds as $nid) { + if (!is_object($nid)) { + $feed = node_load(array('nid' => $nid)); } + else { + $feed = $nid; + } + $feed_nodes[] = $feed; + } + aggregator_feed_retrieve($feed_nodes); + foreach ($feed_nodes as $node) { + aggregator_feed_save($node); } } /** - * Removes all items from a feed. - * - * @param $feed - * An associative array describing the feed to be cleared. + * Builds and initializes a feed object. + * It does not download the feed. it just creates the basic structure + * + * @param $url + * The URL of the feed. + * @param $type + * The content-type of the feed node, this determines the feed configuration. + * @return + * The node-feed object */ -function aggregator_remove($feed) { - $result = db_query('SELECT iid FROM {aggregator_item} WHERE fid = %d', $feed['fid']); - while ($item = db_fetch_object($result)) { - $items[] = "iid = $item->iid"; - } - if (!empty($items)) { - db_query('DELETE FROM {aggregator_category_item} WHERE ' . implode(' OR ', $items)); - } - db_query('DELETE FROM {aggregator_item} WHERE fid = %d', $feed['fid']); - db_query("UPDATE {aggregator_feed} SET checked = 0, etag = '', modified = 0 WHERE fid = %d", $feed['fid']); - drupal_set_message(t('The news items from %site have been removed.', array('%site' => $feed['title']))); +function aggregator_feed_create($url, $type, $node_skeleton = NULL) { + $node = $node_skeleton instanceof stdClass ? $node_skeleton : new stdClass(); + $node->feed = new stdClass(); + $node->type = $type; + $node->feed->url = $url; + return $node; } /** - * Callback function used by the XML parser. - */ -function aggregator_element_start($parser, $name, $attributes) { - global $item, $element, $tag, $items, $channel; - - switch ($name) { - case 'IMAGE': - case 'TEXTINPUT': - case 'CONTENT': - case 'SUMMARY': - case 'TAGLINE': - case 'SUBTITLE': - case 'LOGO': - case 'INFO': - $element = $name; - break; - case 'ID': - if ($element != 'ITEM') { - $element = $name; + * Downloads and parses the feed. populates $feed object and add the items to it. + * It does not save the feed nor the items. + * + * @param $feed_nodes + * The node-feed object (come from aggregator_feed_create). + */ +function aggregator_feed_retrieve(&$feed_nodes) { + // Collect the nodes into groups by content-type + $node_group = array(); + foreach ($feed_nodes as $k => $node) { + if (!isset($node_group[$node->type])) { + $node_group[$node->type] = array(); + } + $node_group[$node->type][$k] = $node; + } + foreach ($node_group as $type => $nodes) { + $parser = variable_get('aggregator_parser_' . $type, FALSE); + $parser_in = $nodes; + $parser_out = module_invoke($parser, 'aggregator_parse', 'parse', $parser_in); + foreach ($parser_out as $k => $feed) { + if (is_string($feed)) { + $url = isset($feed_nodes[$k]->url) ? $feed_nodes[$k]->url : $feed_nodes[$k]->feed->url; + $feed_nodes[$k]->feed = new stdClass(); + $feed_nodes[$k]->feed->url = $url; + $feed_nodes[$k]->feed->error = $feed; } - case 'LINK': - if (!empty($attributes['REL']) && $attributes['REL'] == 'alternate') { - if ($element == 'ITEM') { - $items[$item]['LINK'] = $attributes['HREF']; - } - else { - $channel['LINK'] = $attributes['HREF']; + else { + $feed_nodes[$k]->feed = $feed; + if (isset($feed_nodes[$k]->url) && !isset($feed_nodes[$k]->feed->url)) { + $feed_nodes[$k]->feed->url = $feed_nodes[$k]->url; } } - break; - case 'ITEM': - $element = $name; - $item += 1; - break; - case 'ENTRY': - $element = 'ITEM'; - $item += 1; - break; - } - - $tag = $name; -} - -/** - * Call-back function used by the XML parser. - */ -function aggregator_element_end($parser, $name) { - global $element; - - switch ($name) { - case 'IMAGE': - case 'TEXTINPUT': - case 'ITEM': - case 'ENTRY': - case 'CONTENT': - case 'INFO': - $element = ''; - break; - case 'ID': - if ($element == 'ID') { - $element = ''; - } - } -} - -/** - * Callback function used by the XML parser. - */ -function aggregator_element_data($parser, $data) { - global $channel, $element, $items, $item, $image, $tag; - $items += array($item => array()); - switch ($element) { - case 'ITEM': - $items[$item] += array($tag => ''); - $items[$item][$tag] .= $data; - break; - case 'IMAGE': - case 'LOGO': - $image += array($tag => ''); - $image[$tag] .= $data; - break; - case 'LINK': - if ($data) { - $items[$item] += array($tag => ''); - $items[$item][$tag] .= $data; - } - break; - case 'CONTENT': - $items[$item] += array('CONTENT' => ''); - $items[$item]['CONTENT'] .= $data; - break; - case 'SUMMARY': - $items[$item] += array('SUMMARY' => ''); - $items[$item]['SUMMARY'] .= $data; - break; - case 'TAGLINE': - case 'SUBTITLE': - $channel += array('DESCRIPTION' => ''); - $channel['DESCRIPTION'] .= $data; - break; - case 'INFO': - case 'ID': - case 'TEXTINPUT': - // The sub-element is not supported. However, we must recognize - // it or its contents will end up in the item array. - break; - default: - $channel += array($tag => ''); - $channel[$tag] .= $data; + $feed_nodes[$k]->feed->hash = md5(serialize($feed)); + } } } /** - * Checks a news feed for new items. + * Save or update feed + * It becomes a node and if there are items in the object, the processors are also called * - * @param $feed - * An associative array describing the feed to be refreshed. + * @param $node + * The node-feed object */ -function aggregator_refresh($feed) { - global $channel, $image; - - // Generate conditional GET headers. - $headers = array(); - if ($feed['etag']) { - $headers['If-None-Match'] = $feed['etag']; - } - if ($feed['modified']) { - $headers['If-Modified-Since'] = gmdate('D, d M Y H:i:s', $feed['modified']) . ' GMT'; - } - - // Request feed. - $result = drupal_http_request($feed['url'], $headers); - - // Process HTTP response code. - switch ($result->code) { - case 304: - db_query('UPDATE {aggregator_feed} SET checked = %d WHERE fid = %d', time(), $feed['fid']); - drupal_set_message(t('There is no new syndicated content from %site.', array('%site' => $feed['title']))); - break; - case 301: - $feed['url'] = $result->redirect_url; - watchdog('aggregator', 'Updated URL for feed %title to %url.', array('%title' => $feed['title'], '%url' => $feed['url'])); - // Do not break here. - case 200: - case 302: - case 307: - // Filter the input data. - if (aggregator_parse_feed($result->data, $feed)) { - $modified = empty($result->headers['Last-Modified']) ? 0 : strtotime($result->headers['Last-Modified']); - - // Prepare the channel data. - foreach ($channel as $key => $value) { - $channel[$key] = trim($value); +function aggregator_feed_save($node) { + if (isset($node->feed->error)) { + watchdog('aggregator', 'The feed from %site seems to be broken, due to "%error".', array('%site' => $node->title, '%error' => $node->feed->error), WATCHDOG_WARNING); + drupal_set_message(t('The feed from %site seems to be broken, because of error "%error".', array('%site' => $node->title, '%error' => $node->feed->error))); + } + $hash_array = db_fetch_array(db_query("SELECT hash FROM {aggregator_feed} WHERE nid = %d", $node->nid)); + $hash = is_array($hash_array) ? array_pop($hash_array) : NULL; + if (empty($hash)) { + db_query("INSERT INTO {aggregator_feed} (nid, url, link, image, checked, hash) VALUES (%d, '%s', '%s', '%s', %d, '%s')", $node->nid, $node->feed->url, isset($node->feed->link) ? $node->feed->link : '', isset($node->feed->image) ? $node->feed->image : '', time(), isset($node->feed->hash) ? $node->feed->hash : ''); + } + else { + db_query("UPDATE {aggregator_feed} SET url = '%s', link = '%s', image = '%s', checked = '%s', hash = '%s' WHERE nid = %d", $node->feed->url, isset($node->feed->link) ? $node->feed->link : '', isset($node->feed->image) ? $node->feed->image : '', time(), isset($node->feed->hash) ? $node->feed->hash : '', $node->nid); + } + if (!isset($node->feed->error)) { + if ($hash != $node->feed->hash) { + $processors = variable_get('aggregator_processor_' . $node->type, array()); + $deduper = variable_get('aggregator_deduper_' . $node->type, FALSE); + foreach ($processors as $processor) { + if (count($processors) == 1 && $deduper != FALSE) { + module_invoke($deduper, 'aggregator_process', 'unique', $node); } - - // Prepare the image data (if any). - foreach ($image as $key => $value) { - $image[$key] = trim($value); + else { + module_invoke($processor, 'aggregator_process', 'unique', $node); } - - if (!empty($image['LINK']) && !empty($image['URL']) && !empty($image['TITLE'])) { - // TODO: we should really use theme_image() here, but that only works with - // local images. It won't work with images fetched with a URL unless PHP version > 5. - $image = '' . check_plain($image['TITLE']) . ''; + $items = module_invoke($processor, 'aggregator_process', 'save', $node); + if ($items == 0 && is_numeric($items)) { + drupal_set_message(t('There is no new syndicated content from %site.', array('%site' => $node->title))); } - else { - $image = NULL; + else if(is_numeric($items)) { + watchdog('aggregator', 'There is new syndicated content from %site.', array('%site' => $node->title)); + drupal_set_message(t('There is new syndicated content from %site.', array('%site' => $node->title))); } - - $etag = empty($result->headers['ETag']) ? '' : $result->headers['ETag']; - // Update the feed data. - db_query("UPDATE {aggregator_feed} SET url = '%s', checked = %d, link = '%s', description = '%s', image = '%s', etag = '%s', modified = %d WHERE fid = %d", $feed['url'], time(), $channel['LINK'], $channel['DESCRIPTION'], $image, $etag, $modified, $feed['fid']); - - // Clear the cache. - cache_clear_all(); - - watchdog('aggregator', 'There is new syndicated content from %site.', array('%site' => $feed['title'])); - drupal_set_message(t('There is new syndicated content from %site.', array('%site' => $feed['title']))); - break; - } - $result->error = t('feed not parseable'); - // Do not break here.. - default: - watchdog('aggregator', 'The feed from %site seems to be broken, due to "%error".', array('%site' => $feed['title'], '%error' => $result->code . ' ' . $result->error), WATCHDOG_WARNING); - drupal_set_message(t('The feed from %site seems to be broken, because of error "%error".', array('%site' => $feed['title'], '%error' => $result->code . ' ' . $result->error))); - module_invoke('system', 'check_http_request'); - } -} - -/** - * Parse the W3C date/time format, a subset of ISO 8601. - * - * PHP date parsing functions do not handle this format. - * See http://www.w3.org/TR/NOTE-datetime for more information. - * Originally from MagpieRSS (http://magpierss.sourceforge.net/). - * - * @param $date_str - * A string with a potentially W3C DTF date. - * @return - * A timestamp if parsed successfully or FALSE if not. - */ -function aggregator_parse_w3cdtf($date_str) { - if (preg_match('/(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2})(:(\d{2}))?(?:([-+])(\d{2}):?(\d{2})|(Z))?/', $date_str, $match)) { - list($year, $month, $day, $hours, $minutes, $seconds) = array($match[1], $match[2], $match[3], $match[4], $match[5], $match[6]); - // Calculate the epoch for current date assuming GMT. - $epoch = gmmktime($hours, $minutes, $seconds, $month, $day, $year); - if ($match[10] != 'Z') { // Z is zulu time, aka GMT - list($tz_mod, $tz_hour, $tz_min) = array($match[8], $match[9], $match[10]); - // Zero out the variables. - if (!$tz_hour) { - $tz_hour = 0; - } - if (!$tz_min) { - $tz_min = 0; - } - $offset_secs = (($tz_hour * 60) + $tz_min) * 60; - // Is timezone ahead of GMT? If yes, subtract offset. - if ($tz_mod == '+') { - $offset_secs *= -1; } - $epoch += $offset_secs; } - return $epoch; - } - else { - return FALSE; + else { + return FALSE; + } } } /** - * Parse a feed and store its items. + * Format an individual feed item for display in the block. * - * @param $data - * The feed data. + * @param $item + * The item to be displayed. * @param $feed - * An associative array describing the feed to be parsed. + * Not used. * @return - * FALSE on error, TRUE otherwise. + * The item HTML. + * @ingroup themeable */ -function aggregator_parse_feed(&$data, $feed) { - global $items, $image, $channel; - - // Unset the global variables before we use them. - unset($GLOBALS['element'], $GLOBALS['item'], $GLOBALS['tag']); - $items = array(); - $image = array(); - $channel = array(); - - // Parse the data. - $xml_parser = drupal_xml_parser_create($data); - xml_set_element_handler($xml_parser, 'aggregator_element_start', 'aggregator_element_end'); - xml_set_character_data_handler($xml_parser, 'aggregator_element_data'); - - if (!xml_parse($xml_parser, $data, 1)) { - watchdog('aggregator', 'The feed from %site seems to be broken, due to an error "%error" on line %line.', array('%site' => $feed['title'], '%error' => xml_error_string(xml_get_error_code($xml_parser)), '%line' => xml_get_current_line_number($xml_parser)), WATCHDOG_WARNING); - drupal_set_message(t('The feed from %site seems to be broken, because of error "%error" on line %line.', array('%site' => $feed['title'], '%error' => xml_error_string(xml_get_error_code($xml_parser)), '%line' => xml_get_current_line_number($xml_parser))), 'error'); - return FALSE; - } - xml_parser_free($xml_parser); - - // We reverse the array such that we store the first item last, and the last - // item first. In the database, the newest item should be at the top. - $items = array_reverse($items); - - // Initialize variables. - $title = $link = $author = $description = $guid = NULL; - foreach ($items as $item) { - unset($title, $link, $author, $description, $guid); - - // Prepare the item: - foreach ($item as $key => $value) { - $item[$key] = trim($value); - } - - // Resolve the item's title. If no title is found, we use up to 40 - // characters of the description ending at a word boundary, but not - // splitting potential entities. - if (!empty($item['TITLE'])) { - $title = $item['TITLE']; - } - elseif (!empty($item['DESCRIPTION'])) { - $title = preg_replace('/^(.*)[^\w;&].*?$/', "\\1", truncate_utf8($item['DESCRIPTION'], 40)); - } - else { - $title = ''; - } - - // Resolve the items link. - if (!empty($item['LINK'])) { - $link = $item['LINK']; - } - else { - $link = $feed['link']; - } - $guid = isset($item['GUID']) ? $item['GUID'] : ''; - - // Atom feeds have a CONTENT and/or SUMMARY tag instead of a DESCRIPTION tag. - if (!empty($item['CONTENT:ENCODED'])) { - $item['DESCRIPTION'] = $item['CONTENT:ENCODED']; - } - elseif (!empty($item['SUMMARY'])) { - $item['DESCRIPTION'] = $item['SUMMARY']; - } - elseif (!empty($item['CONTENT'])) { - $item['DESCRIPTION'] = $item['CONTENT']; - } - - // Try to resolve and parse the item's publication date. If no date is - // found, use the current date instead. - $date = 'now'; - foreach (array('PUBDATE', 'DC:DATE', 'DCTERMS:ISSUED', 'DCTERMS:CREATED', 'DCTERMS:MODIFIED', 'ISSUED', 'CREATED', 'MODIFIED', 'PUBLISHED', 'UPDATED') as $key) { - if (!empty($item[$key])) { - $date = $item[$key]; - break; - } - } - - $timestamp = strtotime($date); // As of PHP 5.1.0, strtotime returns FALSE on failure instead of -1. - - if ($timestamp <= 0) { - $timestamp = aggregator_parse_w3cdtf($date); // Aggregator_parse_w3cdtf() returns FALSE on failure. - if (!$timestamp) { - // Better than nothing. - $timestamp = time(); - } - } +function theme_aggregator_block_item($item, $feed = 0) { + global $user; - // Save this item. Try to avoid duplicate entries as much as possible. If - // we find a duplicate entry, we resolve it and pass along its ID is such - // that we can update it if needed. - if (!empty($guid)) { - $entry = db_fetch_object(db_query("SELECT iid FROM {aggregator_item} WHERE fid = %d AND guid = '%s'", $feed['fid'], $guid)); - } - else if ($link && $link != $feed['link'] && $link != $feed['url']) { - $entry = db_fetch_object(db_query("SELECT iid FROM {aggregator_item} WHERE fid = %d AND link = '%s'", $feed['fid'], $link)); - } - else { - $entry = db_fetch_object(db_query("SELECT iid FROM {aggregator_item} WHERE fid = %d AND title = '%s'", $feed['fid'], $title)); + $output = ''; + if ($user->uid && module_exists('blog') && user_access('create blog entries')) { + if ($image = theme('image', 'misc/blog.png', t('blog it'), t('blog it'))) { + $output .= '
' . l($image, 'node/add/blog', array('attributes' => array('title' => t('Comment on this news item in your personal blog.'), 'class' => 'blog-it'), 'query' => "iid=$item->iid", 'html' => TRUE)) . '
'; } - $item += array('AUTHOR' => '', 'DESCRIPTION' => ''); - aggregator_save_item(array('iid' => (isset($entry->iid) ? $entry->iid: ''), 'fid' => $feed['fid'], 'timestamp' => $timestamp, 'title' => $title, 'link' => $link, 'author' => $item['AUTHOR'], 'description' => $item['DESCRIPTION'], 'guid' => $guid)); } - // Remove all items that are older than flush item timer. - $age = time() - variable_get('aggregator_clear', 9676800); - $result = db_query('SELECT iid FROM {aggregator_item} WHERE fid = %d AND timestamp < %d', $feed['fid'], $age); - - $items = array(); - $num_rows = FALSE; - while ($item = db_fetch_object($result)) { - $items[] = $item->iid; - $num_rows = TRUE; - } - if ($num_rows) { - db_query('DELETE FROM {aggregator_category_item} WHERE iid IN (' . implode(', ', $items) . ')'); - db_query('DELETE FROM {aggregator_item} WHERE fid = %d AND timestamp < %d', $feed['fid'], $age); - } + // Display the external link to the item. + $output .= '' . check_plain($item->title) . "\n"; - return TRUE; + return $output; } /** - * Add/edit/delete an aggregator item. + * Page callbacks for refreshing a feed. * - * @param $edit - * An associative array describing the item to be added/edited/deleted. - */ -function aggregator_save_item($edit) { - if ($edit['iid'] && $edit['title']) { - db_query("UPDATE {aggregator_item} SET title = '%s', link = '%s', author = '%s', description = '%s', guid = '%s', timestamp = %d WHERE iid = %d", $edit['title'], $edit['link'], $edit['author'], $edit['description'], $edit['guid'], $edit['timestamp'], $edit['iid']); + * @param $node + * Feed node or array of feed nodes. + * @param $destination_path + * Jumps here after the refresh. + */ +function aggregator_refresh_page($node, $destination_path = NULL) { + $feed_nodes = array($node); + aggregator_feed_refresh($feed_nodes); + if ($destination_path) { + drupal_goto($destination_path); } - elseif ($edit['iid']) { - db_query('DELETE FROM {aggregator_item} WHERE iid = %d', $edit['iid']); - db_query('DELETE FROM {aggregator_category_item} WHERE iid = %d', $edit['iid']); - } - elseif ($edit['title'] && $edit['link']) { - db_query("INSERT INTO {aggregator_item} (fid, title, link, author, description, timestamp, guid) VALUES (%d, '%s', '%s', '%s', '%s', %d, '%s')", $edit['fid'], $edit['title'], $edit['link'], $edit['author'], $edit['description'], $edit['timestamp'], $edit['guid']); - $edit['iid'] = db_last_insert_id('aggregator_item', 'iid'); - // file the items in the categories indicated by the feed - $categories = db_query('SELECT cid FROM {aggregator_category_feed} WHERE fid = %d', $edit['fid']); - while ($category = db_fetch_object($categories)) { - db_query('INSERT INTO {aggregator_category_item} (cid, iid) VALUES (%d, %d)', $category->cid, $edit['iid']); - } + else { + drupal_goto('node/'. $node->nid); } } /** - * Load an aggregator feed. - * - * @param $fid - * The feed id. + * Tells if the given module is enabled for that content-type (as parser or as processor). + * + * @param $module + * The name of the module. + * @param $type + * The name of the content-type. * @return - * An associative array describing the feed. + * TRUE if enabled, FALSE if disabled. */ -function aggregator_feed_load($fid) { - static $feeds; - if (!isset($feeds[$fid])) { - $feeds[$fid] = db_fetch_array(db_query('SELECT * FROM {aggregator_feed} WHERE fid = %d', $fid)); +function aggregator_is_enabled($module, $type) { + $types = node_get_types(); + $type = str_replace('-', '_', $type); + if ($module == variable_get('aggregator_parser_' . $types[$type]->type, '')) { + return TRUE; } - - return $feeds[$fid]; + $processors = array_values(variable_get('aggregator_processor_' . $types[$type]->type, array())); + if (in_array($module, $processors, TRUE)) { + return TRUE; + } + return FALSE; } + /** - * Load an aggregator category. - * - * @param $cid - * The category id. - * @return - * An associative array describing the category. + * Determines if the current user can refresh the feed or not. */ -function aggregator_category_load($cid) { - static $categories; - if (!isset($categories[$cid])) { - $categories[$cid] = db_fetch_array(db_query('SELECT * FROM {aggregator_category} WHERE cid = %d', $cid)); +function _aggregator_perm_refresh($node) { + global $user; + if (variable_get('aggregator_feed_' . $node->type, FALSE) && isset($node->feed) && (user_access("edit any $node->type content") || (user_access("edit own $node->type content") && $node->uid == $user->uid))) { + return TRUE; } + return FALSE; +} - return $categories[$cid]; +/** + * Checks for time limits in cron processing. + */ +function _aggregator_cron_time() { + static $time_limit; + $execute_percentage = 0.5; + if (!$time_limit) { + $time_limit = time() + ($execute_percentage / 100) * ini_get('max_execution_time'); + // However, check for left time, maybe some other cron processing already occured. + $time_limit = min($time_limit, variable_get('cron_semaphore', 0) + ini_get('max_execution_time')); + } + return max($time_limit - time(), 0); } /** - * Format an individual feed item for display in the block. - * + * Decides if the item is new or not. + * The decision is based on the original URL of the article. + * This is the only simple and somewhat reliable way to do. + * @todo: I'm not sure if I agree with that :) it can actually get more complex + * and depending on what you're trying to aggregate you can use many things in your feed item to + * determine duplicity - hence: + * @todo: + * We might want to go back to a hook_aggregator_process operation 'unique' that + * allows any enabled processor to do additional duplicate checks. + * Another option, but less attractive, could be to make it the processor's responsibility to + * provide an API for deduping to other modules. + * Lots of feed use the guid totally wrong. + * + * @param $nid + * Feed-node * @param $item - * The item to be displayed. - * @param $feed - * Not used. + * Feed item * @return - * The item HTML. - * @ingroup themeable + * TRUE if it is a new item, FALSE if it is already in the database. + * */ -function theme_aggregator_block_item($item, $feed = 0) { - global $user; - - $output = ''; - if ($user->uid && module_exists('blog') && user_access('create blog entries')) { - if ($image = theme('image', 'misc/blog.png', t('blog it'), t('blog it'))) { - $output .= '
' . l($image, 'node/add/blog', array('attributes' => array('title' => t('Comment on this news item in your personal blog.'), 'class' => 'blog-it'), 'query' => "iid=$item->iid", 'html' => TRUE)) . '
'; - } - } - - // Display the external link to the item. - $output .= '' . check_plain($item->title) . "\n"; - - return $output; +function _aggregator_unique($nid, $item) { + $result = db_fetch_array(db_query("SELECT COUNT(*) as same FROM {aggregator_item} WHERE link = '%s' AND nid = %d", $item->link, $nid)); + return $result['same'] == 0 ? TRUE : FALSE; } /** - * Safely render HTML content, as allowed. - * - * @param $value - * The content to be filtered. - * @return - * The filtered content. + * Collect the terms from all the aggregator_light-enabled vocabularies */ -function aggregator_filter_xss($value) { - return filter_xss($value, preg_split('/\s+|<|>/', variable_get('aggregator_allowed_html_tags', '