diff -urp Downloads/feeds/feeds.install Sites/leadership.businessspectator.dev/sites/all/modules/contrib/feeds/feeds.install --- Downloads/feeds/feeds.install 2010-09-26 02:07:50.000000000 +1000 +++ Sites/leadership.businessspectator.dev/sites/all/modules/contrib/feeds/feeds.install 2010-11-19 10:01:59.000000000 +1100 @@ -130,6 +130,18 @@ function feeds_schema() { 'default' => '', 'description' => t('The hash of the item.'), ), + 'touched' => array( + 'type' => 'int', + 'not null' => TRUE, + 'default' => 0, + 'description' => t('Mark when a feed item has last been touched, even if not updated.'), + ), + 'orphaned' => array( + 'type' => 'int', + 'not null' => TRUE, + 'default' => 0, + 'description' => t('Mark a feed item as orphaned, ready to be deleted or otherwise dealt with.'), + ), ), 'primary key' => array('nid'), 'indexes' => array( @@ -567,3 +579,29 @@ function feeds_update_6013() { variable_set('feeds_reschedule', TRUE); return array(); } + +/** + * Add touched and orphaned columns to feeds_node_item. + */ +function feeds_update_60014() { + $ret = array(); + + $touched = array( + 'type' => 'int', + 'not null' => TRUE, + 'default' => 0, + 'description' => t('Mark when a feed item has last been touched, even if not updated.'), + ); + + $orphaned = array( + 'type' => 'int', + 'not null' => TRUE, + 'default' => 0, + 'description' => t('Mark a feed item as orphaned, ready to be deleted or otherwise dealt with.'), + ); + + db_add_field($ret, 'feeds_node_item', 'touched', $touched); + db_add_field($ret, 'feeds_node_item', 'orphaned', $orphaned); + + return $ret; +} \ No newline at end of file diff -urp Downloads/feeds/includes/FeedsBatch.inc Sites/leadership.businessspectator.dev/sites/all/modules/contrib/feeds/includes/FeedsBatch.inc --- Downloads/feeds/includes/FeedsBatch.inc 2010-10-26 09:43:03.000000000 +1100 +++ Sites/leadership.businessspectator.dev/sites/all/modules/contrib/feeds/includes/FeedsBatch.inc 2010-11-19 10:01:59.000000000 +1100 @@ -150,6 +150,9 @@ class FeedsImportBatch extends FeedsBatc public $feed_nid; public $created; public $updated; + public $unpublished; + public $deleted; + public $started; public function __construct($raw = '', $feed_nid = 0) { parent::__construct(); @@ -166,6 +169,7 @@ class FeedsImportBatch extends FeedsBatc $this->feed_nid = $feed_nid; $this->created = 0; $this->updated = 0; + $this->started = time(); } /** diff -urp Downloads/feeds/includes/FeedsSource.inc Sites/leadership.businessspectator.dev/sites/all/modules/contrib/feeds/includes/FeedsSource.inc --- Downloads/feeds/includes/FeedsSource.inc 2010-09-27 03:39:45.000000000 +1000 +++ Sites/leadership.businessspectator.dev/sites/all/modules/contrib/feeds/includes/FeedsSource.inc 2010-11-25 12:46:52.000000000 +1100 @@ -194,6 +194,26 @@ class FeedsSource extends FeedsConfigura return $result; } + public function clearOrphans() { + try { + if (!$this->batch || !($this->batch instanceof FeedsClearBatch)) { + $this->batch = new FeedsClearBatch(); + } + $this->importer->processor->clearOrphans($this->batch, $this); + $result = $this->batch->getProgress(); + if ($result == FEEDS_BATCH_COMPLETE) { + unset($this->batch); + } + } + catch (Exception $e) { + unset($this->batch); + $this->save(); + throw $e; + } + $this->save(); + return $result; + } + /** * Schedule this source. */ diff -urp Downloads/feeds/plugins/FeedsNodeProcessor.inc Sites/leadership.businessspectator.dev/sites/all/modules/contrib/feeds/plugins/FeedsNodeProcessor.inc --- Downloads/feeds/plugins/FeedsNodeProcessor.inc 2010-10-26 09:43:03.000000000 +1100 +++ Sites/leadership.businessspectator.dev/sites/all/modules/contrib/feeds/plugins/FeedsNodeProcessor.inc 2010-11-25 12:48:57.000000000 +1100 @@ -15,6 +15,12 @@ define('FEEDS_NODE_SKIP_EXISTING', 0); define('FEEDS_NODE_REPLACE_EXISTING', 1); define('FEEDS_NODE_UPDATE_EXISTING', 2); +define('FEEDS_SYNC_REMOVED_IGNORE', 0); +define('FEEDS_SYNC_REMOVED_UNPUBLISH', 1); +define('FEEDS_SYNC_REMOVED_DELETE', 3); + +define('FEEDS_NODE_ORPHANED', 1); + /** * Creates nodes from feed items. */ @@ -38,6 +44,8 @@ class FeedsNodeProcessor extends FeedsPr // Only proceed if item has actually changed. $hash = $this->hash($item); if (!empty($nid) && $hash == $this->getHash($nid)) { + // To be able to take action on items that have been removed, just indicate that this was still in the feed. + db_query("UPDATE {feeds_node_item} SET touched = %d WHERE nid = %d", FEEDS_REQUEST_TIME, $nid); continue; } @@ -69,16 +77,35 @@ class FeedsNodeProcessor extends FeedsPr } // Set messages. + $modified = FALSE; if ($batch->created) { drupal_set_message(format_plural($batch->created, 'Created @number @type node.', 'Created @number @type nodes.', array('@number' => $batch->created, '@type' => node_get_types('name', $this->config['content_type'])))); + $modified = TRUE; } - elseif ($batch->updated) { + if ($batch->updated) { drupal_set_message(format_plural($batch->updated, 'Updated @number @type node.', 'Updated @number @type nodes.', array('@number' => $batch->updated, '@type' => node_get_types('name', $this->config['content_type'])))); + $modified = TRUE; } - else { - drupal_set_message(t('There is no new content.')); + if (!$modified) { + drupal_set_message(t('There have been no content changes.')); } + $batch->setProgress(FEEDS_PROCESSING, FEEDS_BATCH_COMPLETE); + + // Handle nodes whose corresponding feed items have been removed. + // If there are no items in the total then we assume there has been a timeout or other error + if ($this->config['sync'] != FEEDS_SYNC_REMOVED_IGNORE && $batch->getTotal(FEEDS_PROCESSING) > 0) { + $this->flagOrphans($batch, $source); + } + + if ($this->config['sync'] == FEEDS_SYNC_REMOVED_UNPUBLISH) { + // TODO: We'll see what we might do here later. + } + elseif ($this->config['sync'] == FEEDS_SYNC_REMOVED_DELETE) { + // TODO: Confirm this is the right place and way to call this method. + $this->clearOrphans($batch, $source); + } + } /** @@ -111,6 +138,42 @@ class FeedsNodeProcessor extends FeedsPr } /** + * Flag nodes that don't have a corresponding feed item in the feed anymore. + */ + public function flagOrphans(FeedsBatch $batch, FeedsSource $source) { + db_query("UPDATE {feeds_node_item} SET orphaned = 1 WHERE feed_nid = %d AND touched < %d", $source->feed_nid, FEEDS_REQUEST_TIME); + } + + /** + * Implementation of FeedsProcessor::clearOrphans(). + */ + public function clearOrphans(FeedsBatch $batch, FeedsSource $source) { + if (!$batch->getTotal(FEEDS_CLEARING)) { + $total = db_result(db_query("SELECT COUNT(nid) FROM {feeds_node_item} WHERE id = '%s' AND feed_nid = %d AND orphaned = %d", $source->id, $source->feed_nid, FEEDS_NODE_ORPHANED)); + $batch->setTotal(FEEDS_CLEARING, $total); + } + $result = db_query_range("SELECT nid FROM {feeds_node_item} WHERE id = '%s' AND feed_nid = %d AND orphaned = %d", $source->id, $source->feed_nid, FEEDS_NODE_ORPHANED, 0, variable_get('feeds_node_batch_size', FEEDS_NODE_BATCH_SIZE)); + while ($node = db_fetch_object($result)) { + _feeds_node_delete($node->nid); + $batch->deleted++; + } + if (db_result(db_query_range("SELECT nid FROM {feeds_node_item} WHERE id = '%s' AND feed_nid = %d AND orphaned = %d", $source->id, $source->feed_nid, FEEDS_NODE_ORPHANED, 0, 1))) { + $batch->setProgress(FEEDS_CLEARING, $batch->deleted); + return; + } + + // Set message. + //drupal_get_messages('status', FALSE); + if ($batch->deleted) { + drupal_set_message(format_plural($batch->deleted, 'Deleted @number node.', 'Deleted @number nodes.', array('@number' => $batch->deleted))); + } + else { + drupal_set_message(t('There are no orphaned nodes to be deleted.')); + } + $batch->setProgress(FEEDS_CLEARING, FEEDS_BATCH_COMPLETE); + } + + /** * Implement expire(). */ public function expire($time = NULL) { @@ -150,6 +213,7 @@ class FeedsNodeProcessor extends FeedsPr 'expire' => FEEDS_EXPIRE_NEVER, 'mappings' => array(), 'author' => 0, + 'sync' => FEEDS_SYNC_REMOVED_IGNORE, ); } @@ -205,6 +269,17 @@ class FeedsNodeProcessor extends FeedsPr ), '#default_value' => $this->config['update_existing'], ); + $form['sync'] = array( + '#type' => 'radios', + '#title' => t('Keep nodes and feed items in sync'), + '#description' => t('Select how nodes corresponding to removed feed items should be handled. This applies if you want to only display nodes that map to items currently in the feed.'), + '#options' => array( + FEEDS_SYNC_REMOVED_IGNORE => 'Do nothing', + FEEDS_SYNC_REMOVED_UNPUBLISH => 'Unpublish the node', + FEEDS_SYNC_REMOVED_DELETE => 'Delete the node', + ), + '#default_value' => $this->config['sync'], + ); return $form; } @@ -351,6 +426,7 @@ class FeedsNodeProcessor extends FeedsPr if ($populate) { $node->type = $this->config['content_type']; $node->changed = FEEDS_REQUEST_TIME; + $node->status = 1; $node->format = $this->config['input_format']; $node->feeds_node_item = new stdClass(); $node->feeds_node_item->id = $this->id; @@ -358,6 +434,7 @@ class FeedsNodeProcessor extends FeedsPr $node->feeds_node_item->feed_nid = $feed_nid; $node->feeds_node_item->url = ''; $node->feeds_node_item->guid = ''; + $node->feeds_node_item->touched = FEEDS_REQUEST_TIME; } static $included;