Date: Fri, 11 Mar 2011 20:29:36 +1000 Subject: [PATCH] Merged changes from http://drupal.org/node/661314#comment-4127484 into latest dev version. --- feeds.install | 38 ++++++++++++++++++++++ includes/FeedsBatch.inc | 1 + includes/FeedsImporter.inc | 2 +- plugins/FeedsNodeProcessor.inc | 68 +++++++++++++++++++++++++++++++++------- 4 files changed, 96 insertions(+), 13 deletions(-) diff --git a/feeds.install b/feeds.install index 80c4b5e..a1bf1fb 100644 --- a/feeds.install +++ b/feeds.install @@ -129,6 +129,18 @@ function feeds_schema() { 'default' => '', 'description' => t('The hash of the item.'), ), + 'touched' => array( + 'type' => 'int', + 'not null' => TRUE, + 'default' => 0, + 'description' => t('Mark when a feed item has last been touched, even if not updated.'), + ), + 'orphaned' => array( + 'type' => 'int', + 'not null' => TRUE, + 'default' => 0, + 'description' => t('Mark a feed item as orphaned, ready to be deleted.'), + ), ), 'primary key' => array('nid'), 'indexes' => array( @@ -627,3 +639,29 @@ function feeds_update_6014() { return $ret; } + +/** + * Add touched and orphaned columns to feeds_node_item. + */ +function feeds_update_6015() { + $ret = array(); + + $touched = array( + 'type' => 'int', + 'not null' => TRUE, + 'default' => 0, + 'description' => t('Mark when a feed item has last been touched, even if not updated.'), + ); + + $orphaned = array( + 'type' => 'int', + 'not null' => TRUE, + 'default' => 0, + 'description' => t('Mark a feed item as orphaned, ready to be deleted.'), + ); + + db_add_field($ret, 'feeds_node_item', 'touched', $touched); + db_add_field($ret, 'feeds_node_item', 'orphaned', $orphaned); + + return $ret; +} \ No newline at end of file diff --git a/includes/FeedsBatch.inc b/includes/FeedsBatch.inc index 5971675..ca54f8d 100644 --- a/includes/FeedsBatch.inc +++ b/includes/FeedsBatch.inc @@ -19,6 +19,7 @@ class FeedsBatch { public function __construct() { $this->total = array(); $this->progress = array(); + $this->started = FEEDS_REQUEST_TIME; } /** diff --git a/includes/FeedsImporter.inc b/includes/FeedsImporter.inc index e3d8591..4957088 100644 --- a/includes/FeedsImporter.inc +++ b/includes/FeedsImporter.inc @@ -87,7 +87,7 @@ class FeedsImporter extends FeedsConfigurable { 'period' => 0, 'periodic' => TRUE, ); - if (FEEDS_EXPIRE_NEVER != $this->processor->expiryTime()) { + if (FEEDS_EXPIRE_NEVER != $this->processor->expiryTime() || $this->processor->config['delete_orphans']) { $job['period'] = 3600; job_scheduler()->set($job); } diff --git a/plugins/FeedsNodeProcessor.inc b/plugins/FeedsNodeProcessor.inc index 8c3edf7..ed2ffdc 100644 --- a/plugins/FeedsNodeProcessor.inc +++ b/plugins/FeedsNodeProcessor.inc @@ -14,6 +14,8 @@ define('FEEDS_NODE_SKIP_EXISTING', 0); define('FEEDS_NODE_REPLACE_EXISTING', 1); define('FEEDS_NODE_UPDATE_EXISTING', 2); +define('FEEDS_NODE_ORPHANED', 1); + /** * Creates nodes from feed items. */ @@ -34,10 +36,13 @@ class FeedsNodeProcessor extends FeedsProcessor { // Create/update if item does not exist or update existing is enabled. if (!($nid = $this->existingItemId($batch, $source)) || ($this->config['update_existing'] != FEEDS_SKIP_EXISTING)) { - // Only proceed if item has actually changed. $hash = $this->hash($item); - if (!empty($nid) && $hash == $this->getHash($nid)) { - continue; + if (!empty($nid)) { + // To be able to take action on items that have been removed, just indicate that this was still in the feed. + db_query("UPDATE {feeds_node_item} SET touched = %d WHERE nid = %d", FEEDS_REQUEST_TIME, $nid); + + // Only proceed if item has actually changed. + if ($hash == $this->getHash($nid)) continue; } $node = $this->buildNode($nid, $source->feed_nid); @@ -80,16 +85,25 @@ class FeedsNodeProcessor extends FeedsProcessor { } // Set messages. + $modified = FALSE; if ($batch->created) { drupal_set_message(format_plural($batch->created, 'Created @number @type node.', 'Created @number @type nodes.', array('@number' => $batch->created, '@type' => node_get_types('name', $this->config['content_type'])))); + $modified = TRUE; } - elseif ($batch->updated) { + if ($batch->updated) { drupal_set_message(format_plural($batch->updated, 'Updated @number @type node.', 'Updated @number @type nodes.', array('@number' => $batch->updated, '@type' => node_get_types('name', $this->config['content_type'])))); + $modified = TRUE; } - else { - drupal_set_message(t('There is no new content.')); + if (!$modified) { + drupal_set_message(t('There have been no content changes.')); } + $batch->setProgress(FEEDS_PROCESSING, FEEDS_BATCH_COMPLETE); + + // Handle nodes whose corresponding feed items have been removed. + if ($this->config['delete_orphans']) { + $this->flagOrphans($batch, $source); + } } /** @@ -122,20 +136,42 @@ class FeedsNodeProcessor extends FeedsProcessor { } /** + * Flag nodes that don't have a corresponding feed item in the feed anymore. + */ + public function flagOrphans($batch, $source) { + db_query("UPDATE {feeds_node_item} SET orphaned = %d WHERE id = '%s'", 0, $source->id); + db_query("UPDATE {feeds_node_item} SET orphaned = %d WHERE id = '%s' AND touched < %d", FEEDS_NODE_ORPHANED, $source->id, $batch->started); + drupal_set_message(t('Flagged %num orphaned nodes for deletion.', array('%num' => db_affected_rows()))); + } + + /** * Implement expire(). */ public function expire($time = NULL) { + $or = array(); + $args = array(); + if ($time === NULL) { $time = $this->expiryTime(); } - if ($time == FEEDS_EXPIRE_NEVER) { - return; + if ($time != FEEDS_EXPIRE_NEVER) { + $or[] = 'n.created < %d'; + $args[] = FEEDS_REQUEST_TIME - $time; + } + + if ($this->config['delete_orphans']) { + $or[] = 'orphaned = %d'; + $args[] = FEEDS_NODE_ORPHANED; } - $result = db_query_range("SELECT n.nid FROM {node} n JOIN {feeds_node_item} fni ON n.nid = fni.nid WHERE fni.id = '%s' AND n.created < %d", $this->id, FEEDS_REQUEST_TIME - $time, 0, variable_get('feeds_node_batch_size', FEEDS_NODE_BATCH_SIZE)); + + $or = implode(' OR ', $or); + $args = implode(', ', $args); + + $result = db_query_range("SELECT n.nid FROM {node} n JOIN {feeds_node_item} fni ON n.nid = fni.nid WHERE fni.id = '%s' AND ($or)", $this->id, $args, 0, variable_get('feeds_node_batch_size', FEEDS_NODE_BATCH_SIZE)); while ($node = db_fetch_object($result)) { _feeds_node_delete($node->nid); } - if (db_result(db_query_range("SELECT n.nid FROM {node} n JOIN {feeds_node_item} fni ON n.nid = fni.nid WHERE fni.id = '%s' AND n.created < %d", $this->id, FEEDS_REQUEST_TIME - $time, 0, 1))) { + if (db_result(db_query_range("SELECT n.nid FROM {node} n JOIN {feeds_node_item} fni ON n.nid = fni.nid WHERE fni.id = '%s' AND ($or)", $this->id, $args, 0, 1))) { return FEEDS_BATCH_ACTIVE; } return FEEDS_BATCH_COMPLETE; @@ -162,6 +198,7 @@ class FeedsNodeProcessor extends FeedsProcessor { 'mappings' => array(), 'author' => 0, 'authorize' => 0, + 'delete_orphans' => 0, ); } @@ -223,6 +260,12 @@ class FeedsNodeProcessor extends FeedsProcessor { ), '#default_value' => $this->config['update_existing'], ); + $form['delete_orphans'] = array( + '#type' => 'checkbox', + '#title' => t('Delete nodes for missing items'), + '#description' => t('Delete existing nodes if the corresponding item no longer appears in the feed.'), + '#default_value' => $this->config['delete_orphans'], + ); return $form; } @@ -239,10 +282,10 @@ class FeedsNodeProcessor extends FeedsProcessor { } /** - * Reschedule if expiry time changes. + * Reschedule if expiry time changes or deleting orphans gets enabled. */ public function configFormSubmit(&$values) { - if ($this->config['expire'] != $values['expire']) { + if ($this->config['expire'] != $values['expire'] || $this->config['delete_orphans'] != $values['delete_orphans']) { feeds_reschedule($this->id); } parent::configFormSubmit($values); @@ -377,6 +420,7 @@ class FeedsNodeProcessor extends FeedsProcessor { $node->feeds_node_item->feed_nid = $feed_nid; $node->feeds_node_item->url = ''; $node->feeds_node_item->guid = ''; + $node->feeds_node_item->touched = FEEDS_REQUEST_TIME; } static $included; -- 1.7.1