diff --git a/feeds.install b/feeds.install index 24cc424..4de8dad 100644 --- a/feeds.install +++ b/feeds.install @@ -130,6 +130,18 @@ function feeds_schema() { 'default' => '', 'description' => t('The hash of the item.'), ), + 'touched' => array( + 'type' => 'int', + 'not null' => TRUE, + 'default' => 0, + 'description' => t('Mark when a feed item has last been touched, even if not updated.'), + ), + 'orphaned' => array( + 'type' => 'int', + 'not null' => TRUE, + 'default' => 0, + 'description' => t('Mark a feed item as orphaned, ready to be deleted.'), + ), ), 'primary key' => array('nid'), 'indexes' => array( @@ -567,3 +579,29 @@ function feeds_update_6013() { variable_set('feeds_reschedule', TRUE); return array(); } + +/** + * Add touched and orphaned columns to feeds_node_item. + */ +function feeds_update_6015() { + $ret = array(); + + $touched = array( + 'type' => 'int', + 'not null' => TRUE, + 'default' => 0, + 'description' => t('Mark when a feed item has last been touched, even if not updated.'), + ); + + $orphaned = array( + 'type' => 'int', + 'not null' => TRUE, + 'default' => 0, + 'description' => t('Mark a feed item as orphaned, ready to be deleted.'), + ); + + db_add_field($ret, 'feeds_node_item', 'touched', $touched); + db_add_field($ret, 'feeds_node_item', 'orphaned', $orphaned); + + return $ret; +} \ No newline at end of file diff --git a/includes/FeedsBatch.inc b/includes/FeedsBatch.inc index 05ac8df..240e2b4 100644 --- a/includes/FeedsBatch.inc +++ b/includes/FeedsBatch.inc @@ -20,6 +20,7 @@ class FeedsBatch { public function __construct() { $this->total = array(); $this->progress = array(); + $this->started = FEEDS_REQUEST_TIME; } /** diff --git a/includes/FeedsImporter.inc b/includes/FeedsImporter.inc index 587e755..b723b3e 100644 --- a/includes/FeedsImporter.inc +++ b/includes/FeedsImporter.inc @@ -88,8 +88,8 @@ class FeedsImporter extends FeedsConfigurable { 'period' => 0, 'periodic' => TRUE, ); - if (FEEDS_EXPIRE_NEVER != $this->processor->expiryTime()) { - $job['period'] = 3600; + if (FEEDS_EXPIRE_NEVER != $this->processor->expiryTime() || $this->processor->config['delete_orphans']) { + //$job['period'] = 3600; removed this to make the deletion of nodes happen immediatly. job_scheduler()->set($job); } else { diff --git a/libraries/common_syndication_parser.inc b/libraries/common_syndication_parser.inc index 4cd8c5a..ddfc7b8 100644 --- a/libraries/common_syndication_parser.inc +++ b/libraries/common_syndication_parser.inc @@ -412,6 +412,7 @@ function _parser_common_syndication_RSS20_parse($feed_XML) { if (isset($news['guid'])) { $guid = "{$news['guid']}"; + $original_url .= '?guid='. $guid; } if (isset($georss['featureName'])) { diff --git a/plugins/FeedsNodeProcessor.inc b/plugins/FeedsNodeProcessor.inc index fd36254..198af1b 100644 --- a/plugins/FeedsNodeProcessor.inc +++ b/plugins/FeedsNodeProcessor.inc @@ -15,6 +15,8 @@ define('FEEDS_NODE_SKIP_EXISTING', 0); define('FEEDS_NODE_REPLACE_EXISTING', 1); define('FEEDS_NODE_UPDATE_EXISTING', 2); +define('FEEDS_NODE_ORPHANED', 1); + /** * Creates nodes from feed items. */ @@ -32,15 +34,20 @@ class FeedsNodeProcessor extends FeedsProcessor { } while ($item = $batch->shiftItem()) { - + $nid = $this->existingItemId($batch, $source); + if ($this->config['delete_orphans']) { + if (!empty($nid)) { + // To be able to take action on items that have been removed, just indicate that this was still in the feed. + db_query("UPDATE {feeds_node_item} SET touched = %d WHERE nid = %d", FEEDS_REQUEST_TIME, $nid); + } + } // Create/update if item does not exist or update existing is enabled. - if (!($nid = $this->existingItemId($batch, $source)) || ($this->config['update_existing'] != FEEDS_SKIP_EXISTING)) { + if (!($nid) || ($this->config['update_existing'] != FEEDS_SKIP_EXISTING)) { // Only proceed if item has actually changed. $hash = $this->hash($item); if (!empty($nid) && $hash == $this->getHash($nid)) { continue; } - $node = $this->buildNode($nid, $source->feed_nid); $node->feeds_node_item->hash = $hash; @@ -69,16 +76,25 @@ class FeedsNodeProcessor extends FeedsProcessor { } // Set messages. + $modified = FALSE; if ($batch->created) { drupal_set_message(format_plural($batch->created, 'Created @number @type node.', 'Created @number @type nodes.', array('@number' => $batch->created, '@type' => node_get_types('name', $this->config['content_type'])))); + $modified = TRUE; } - elseif ($batch->updated) { + if ($batch->updated) { drupal_set_message(format_plural($batch->updated, 'Updated @number @type node.', 'Updated @number @type nodes.', array('@number' => $batch->updated, '@type' => node_get_types('name', $this->config['content_type'])))); + $modified = TRUE; } - else { - drupal_set_message(t('There is no new content.')); + if (!$modified) { + drupal_set_message(t('There have been no content changes.')); } + $batch->setProgress(FEEDS_PROCESSING, FEEDS_BATCH_COMPLETE); + + // Handle nodes whose corresponding feed items have been removed. + if ($this->config['delete_orphans']) { + $this->flagOrphans($batch, $source); + } } /** @@ -111,20 +127,42 @@ class FeedsNodeProcessor extends FeedsProcessor { } /** + * Flag nodes that don't have a corresponding feed item in the feed anymore. + */ + public function flagOrphans($batch, $source) { + db_query("UPDATE {feeds_node_item} SET orphaned = %d WHERE id = '%s' AND feed_nid = %d", 0, $source->id, $source->feed_nid); + db_query("UPDATE {feeds_node_item} SET orphaned = %d WHERE id = '%s' AND feed_nid = %d AND touched < %d", FEEDS_NODE_ORPHANED, $source->id, $source->feed_nid, $batch->started); + drupal_set_message(t('Flagged %num orphaned nodes for deletion.', array('%num' => db_affected_rows()))); + } + + /** * Implement expire(). */ public function expire($time = NULL) { + $or = array(); + $args = array(); + if ($time === NULL) { $time = $this->expiryTime(); } - if ($time == FEEDS_EXPIRE_NEVER) { - return; + if ($time != FEEDS_EXPIRE_NEVER) { + $or[] = 'n.created < %d'; + $args[] = FEEDS_REQUEST_TIME - $time; } - $result = db_query_range("SELECT n.nid FROM {node} n JOIN {feeds_node_item} fni ON n.nid = fni.nid WHERE fni.id = '%s' AND n.created < %d", $this->id, FEEDS_REQUEST_TIME - $time, 0, variable_get('feeds_node_batch_size', FEEDS_NODE_BATCH_SIZE)); + + if ($this->config['delete_orphans']) { + $or[] = 'orphaned = %d'; + $args[] = FEEDS_NODE_ORPHANED; + } + + $or = implode(' OR ', $or); + $args = implode(', ', $args); + + $result = db_query_range("SELECT n.nid FROM {node} n JOIN {feeds_node_item} fni ON n.nid = fni.nid WHERE fni.id = '%s' AND ($or)", $this->id, $args, 0, variable_get('feeds_node_batch_size', FEEDS_NODE_BATCH_SIZE)); while ($node = db_fetch_object($result)) { _feeds_node_delete($node->nid); } - if (db_result(db_query_range("SELECT n.nid FROM {node} n JOIN {feeds_node_item} fni ON n.nid = fni.nid WHERE fni.id = '%s' AND n.created < %d", $this->id, FEEDS_REQUEST_TIME - $time, 0, 1))) { + if (db_result(db_query_range("SELECT n.nid FROM {node} n JOIN {feeds_node_item} fni ON n.nid = fni.nid WHERE fni.id = '%s' AND ($or)", $this->id, $args, 0, 1))) { return FEEDS_BATCH_ACTIVE; } return FEEDS_BATCH_COMPLETE; @@ -150,6 +188,7 @@ class FeedsNodeProcessor extends FeedsProcessor { 'expire' => FEEDS_EXPIRE_NEVER, 'mappings' => array(), 'author' => 0, + 'delete_orphans' => 0, ); } @@ -205,6 +244,12 @@ class FeedsNodeProcessor extends FeedsProcessor { ), '#default_value' => $this->config['update_existing'], ); + $form['delete_orphans'] = array( + '#type' => 'checkbox', + '#title' => t('Delete nodes for missing items'), + '#description' => t('Delete existing nodes if the corresponding item no longer appears in the feed.'), + '#default_value' => $this->config['delete_orphans'], + ); return $form; } @@ -221,10 +266,10 @@ class FeedsNodeProcessor extends FeedsProcessor { } /** - * Reschedule if expiry time changes. + * Reschedule if expiry time changes or deleting orphans gets enabled. */ public function configFormSubmit(&$values) { - if ($this->config['expire'] != $values['expire']) { + if ($this->config['expire'] != $values['expire'] || $this->config['delete_orphans'] != $values['delete_orphans']) { feeds_reschedule($this->id); } parent::configFormSubmit($values); @@ -358,6 +403,7 @@ class FeedsNodeProcessor extends FeedsProcessor { $node->feeds_node_item->feed_nid = $feed_nid; $node->feeds_node_item->url = ''; $node->feeds_node_item->guid = ''; + $node->feeds_node_item->touched = FEEDS_REQUEST_TIME; } static $included;