diff --git a/src/Entity/Feed.php b/src/Entity/Feed.php index c42a992..a1bc006 100644 --- a/src/Entity/Feed.php +++ b/src/Entity/Feed.php @@ -379,6 +379,13 @@ class Feed extends ContentEntityBase implements FeedInterface { /** * {@inheritdoc} */ + public function progressCleaning() { + return $this->getState(StateInterface::CLEAN)->progress; + } + + /** + * {@inheritdoc} + */ public function progressClearing() { return $this->getState(StateInterface::CLEAR)->progress; } diff --git a/src/Event/CleanEvent.php b/src/Event/CleanEvent.php new file mode 100644 index 0000000..2ba61bf --- /dev/null +++ b/src/Event/CleanEvent.php @@ -0,0 +1,43 @@ +feed = $feed; + $this->entity = $entity; + } + + /** + * Returns the entity to clean. + * + * @return \Drupal\Core\Entity\EntityInterface + * The entity to clean. + */ + public function getEntity() { + return $this->entity; + } + +} diff --git a/src/Event/FeedsEvents.php b/src/Event/FeedsEvents.php index 9abb2eb..273e8be 100644 --- a/src/Event/FeedsEvents.php +++ b/src/Event/FeedsEvents.php @@ -43,6 +43,11 @@ final class FeedsEvents { const PROCESS = 'feeds.process'; /** + * Fired when cleaning has started. + */ + const CLEAN = 'feeds.clean'; + + /** * Fired before clearing begins. */ const INIT_CLEAR = 'feeds.init_clear'; diff --git a/src/EventSubscriber/LazySubscriber.php b/src/EventSubscriber/LazySubscriber.php index 584e6a5..ce642bb 100644 --- a/src/EventSubscriber/LazySubscriber.php +++ b/src/EventSubscriber/LazySubscriber.php @@ -9,6 +9,8 @@ use Drupal\feeds\Event\FetchEvent; use Drupal\feeds\Event\InitEvent; use Drupal\feeds\Event\ParseEvent; use Drupal\feeds\Event\ProcessEvent; +use Drupal\feeds\Event\CleanEvent; +use Drupal\feeds\Plugin\Type\CleanableInterface; use Drupal\feeds\Plugin\Type\ClearableInterface; use Drupal\feeds\StateInterface; use Symfony\Component\EventDispatcher\EventDispatcherInterface; @@ -92,6 +94,20 @@ class LazySubscriber implements EventSubscriberInterface { ->process($feed, $event->getParserResult(), $feed->getState(StateInterface::PROCESS)); }); break; + + case 'clean': + foreach ($event->getFeed()->getType()->getPlugins() as $plugin) { + if (!$plugin instanceof CleanableInterface) { + continue; + } + + $dispatcher->addListener(FeedsEvents::CLEAN, function (CleanEvent $event) use ($plugin) { + $feed = $event->getFeed(); + $plugin->clean($feed, $event->getEntity(), $feed->getState(StateInterface::CLEAN)); + }); + } + break; + } } diff --git a/src/FeedImportHandler.php b/src/FeedImportHandler.php index 3fdd03f..ff3a4eb 100644 --- a/src/FeedImportHandler.php +++ b/src/FeedImportHandler.php @@ -2,6 +2,8 @@ namespace Drupal\feeds; +use Drupal\Core\Entity\EntityInterface; +use Drupal\feeds\Event\CleanEvent; use Drupal\feeds\Event\FeedsEvents; use Drupal\feeds\Event\FetchEvent; use Drupal\feeds\Event\InitEvent; @@ -45,6 +47,19 @@ class FeedImportHandler extends FeedHandlerBase { $this->doProcess($feed, $item); } } while ($feed->progressImporting() !== StateInterface::BATCH_COMPLETE); + + // Clean up if needed. + // @todo move this logic. + $clean_state = $feed->getState(StateInterface::CLEAN); + if (isset($clean_state->cleanList) && isset($clean_state->entityType)) { + $storage = \Drupal::entityTypeManager()->getStorage($clean_state->entityType); + foreach ($clean_state->cleanList as $entity_id) { + $entity = $storage->load($entity_id); + if ($entity) { + $this->doClean($feed, $entity); + } + } + } } catch (EmptyFeedException $e) { // Not an error. @@ -299,6 +314,19 @@ class FeedImportHandler extends FeedHandlerBase { } /** + * Invokes the clean stage. + * + * @param \Drupal\feeds\FeedInterface $feed + * The feed to fetch. + * @param \Drupal\Core\Entity\EntityInterface $entity + * The entity to apply an action on. + */ + protected function doClean(FeedInterface $feed, EntityInterface $entity) { + $this->dispatchEvent(FeedsEvents::INIT_IMPORT, new InitEvent($feed, 'clean')); + $this->dispatchEvent(FeedsEvents::CLEAN, new CleanEvent($feed, $entity)); + } + + /** * Handles an exception during importing. * * @param \Drupal\feeds\FeedInterface $feed diff --git a/src/FeedInterface.php b/src/FeedInterface.php index 66cac91..7a91e11 100644 --- a/src/FeedInterface.php +++ b/src/FeedInterface.php @@ -195,6 +195,11 @@ interface FeedInterface extends ContentEntityInterface, EntityChangedInterface, public function progressImporting(); /** + * Reports progress on cleaning. + */ + public function progressCleaning(); + + /** * Reports progress on clearing. */ public function progressClearing(); diff --git a/src/Feeds/Processor/EntityProcessorBase.php b/src/Feeds/Processor/EntityProcessorBase.php index 1a874a3..4e0e32e 100644 --- a/src/Feeds/Processor/EntityProcessorBase.php +++ b/src/Feeds/Processor/EntityProcessorBase.php @@ -101,9 +101,20 @@ abstract class EntityProcessorBase extends ProcessorBase implements EntityProces * {@inheritdoc} */ public function process(FeedInterface $feed, ItemInterface $item, StateInterface $state) { + // Initialize clean list if needed. + $clean_state = $feed->getState(StateInterface::CLEAN); + if (!isset($clean_state->cleanList)) { + $this->initCleanList($feed, $clean_state); + } + $existing_entity_id = $this->existingEntityId($feed, $item); $skip_existing = $this->configuration['update_existing'] == static::SKIP_EXISTING; + // If the entity is an existing entity it must be removed from the clean list. + if ($existing_entity_id) { + unset($clean_state->cleanList[$existing_entity_id]); + } + // Bulk load existing entities to save on db queries. if ($skip_existing && $existing_entity_id) { return; @@ -155,6 +166,34 @@ abstract class EntityProcessorBase extends ProcessorBase implements EntityProces } /** + * Initializes the list of entities to clean. + * + * This populates $state->cleanList with all existing entities previously + * imported from the source. + * + * @param \Drupal\feeds\FeedInterface $feed + * The feed to import. + * @param \Drupal\feeds\StateInterface $state + * The state of the clean stage. + */ + protected function initCleanList(FeedInterface $feed, StateInterface $state) { + $state->cleanList = []; + $state->entityType = $this->entityType(); + + // Fill the list only if needed. + if ($this->getConfiguration('update_non_existent') === static::KEEP_NON_EXISTENT) { + return; + } + + $entity_ids = $this->getImportedItemIds($feed); + if (empty($entity_ids)) { + return; + } + + $state->cleanList = array_combine($entity_ids, $entity_ids); + } + + /** * {@inheritdoc} */ public function clean(FeedInterface $feed, EntityInterface $entity, StateInterface $state) { diff --git a/src/Plugin/QueueWorker/FeedRefresh.php b/src/Plugin/QueueWorker/FeedRefresh.php index f2210be..d0ed05b 100644 --- a/src/Plugin/QueueWorker/FeedRefresh.php +++ b/src/Plugin/QueueWorker/FeedRefresh.php @@ -2,6 +2,8 @@ namespace Drupal\feeds\Plugin\QueueWorker; +use Drupal\Core\Entity\EntityInterface; +use Drupal\feeds\Event\CleanEvent; use Drupal\feeds\Event\FeedsEvents; use Drupal\feeds\Event\FetchEvent; use Drupal\feeds\Event\InitEvent; @@ -185,6 +187,20 @@ class FeedRefresh extends FeedQueueWorkerBase { } /** + * Cleans an entity. + * + * @param \Drupal\feeds\FeedInterface $feed + * The feed to perform a clean event on. + * @param \Drupal\Core\Entity\EntityInterface $entity + * The entity to clean. + */ + protected function doClean(FeedInterface $feed, EntityInterface $entity) { + $this->dispatchEvent(FeedsEvents::INIT_IMPORT, new InitEvent($feed, 'clean')); + $this->dispatchEvent(FeedsEvents::CLEAN, new CleanEvent($feed, $entity)); + $feed->saveStates(); + } + + /** * Finalizes the import. */ protected function finish(FeedInterface $feed, FetcherResultInterface $fetcher_result) { diff --git a/src/StateInterface.php b/src/StateInterface.php index 0833821..8305a09 100644 --- a/src/StateInterface.php +++ b/src/StateInterface.php @@ -43,6 +43,13 @@ interface StateInterface { const PROCESS = 'process'; /** + * Denotes the clean stage. + * + * @var string + */ + const CLEAN = 'clean'; + + /** * Denotes the clear stage. * * @var string diff --git a/tests/src/Kernel/Entity/FeedTest.php b/tests/src/Kernel/Entity/FeedTest.php index 706ee73..1ff7e8e 100644 --- a/tests/src/Kernel/Entity/FeedTest.php +++ b/tests/src/Kernel/Entity/FeedTest.php @@ -270,6 +270,14 @@ class FeedTest extends FeedsKernelTestBase { } /** + * @covers ::progressCleaning + */ + public function testProgressCleaning() { + $feed = $this->createFeed($this->feedType->id()); + $this->assertInternalType('float', $feed->progressCleaning()); + } + + /** * @covers ::progressClearing */ public function testProgressClearing() { @@ -293,6 +301,7 @@ class FeedTest extends FeedsKernelTestBase { $this->assertInstanceOf(StateInterface::class, $feed->getState(StateInterface::FETCH)); $this->assertInstanceOf(StateInterface::class, $feed->getState(StateInterface::PARSE)); $this->assertInstanceOf(StateInterface::class, $feed->getState(StateInterface::PROCESS)); + $this->assertInstanceOf(StateInterface::class, $feed->getState(StateInterface::CLEAN)); $this->assertInstanceOf(StateInterface::class, $feed->getState(StateInterface::CLEAR)); } diff --git a/tests/src/Kernel/UpdateNonExistentTest.php b/tests/src/Kernel/UpdateNonExistentTest.php index bdb8458..53921a2 100644 --- a/tests/src/Kernel/UpdateNonExistentTest.php +++ b/tests/src/Kernel/UpdateNonExistentTest.php @@ -33,9 +33,14 @@ class UpdateNonExistentTest extends FeedsKernelTestBase { 'fetcher_configuration' => [ 'allowed_extensions' => 'atom rss rss1 rss2 opml xml', ], + 'processor_configuration' => [ + 'authorize' => FALSE, + 'update_existing' => ProcessorInterface::UPDATE_EXISTING, + 'values' => [ + 'type' => 'article', + ], + ], ]); - // Add mapping to status? - // @todo } /** @@ -75,8 +80,9 @@ class UpdateNonExistentTest extends FeedsKernelTestBase { $node = $this->getNodeByTitle('Egypt, Hamas exchange fire on Gaza frontier, 1 dead - Reuters'); $this->assertFalse($node->isPublished()); - // Manually set the last changed date of this node. + // Manually set title and the last changed date of this node. $update_date = $node->getChangedTime() - 100; + $node->setTitle('Lorem'); $node->changed = $update_date; $node->save(); @@ -102,6 +108,7 @@ class UpdateNonExistentTest extends FeedsKernelTestBase { $feed->import(); $storage->resetCache([$node->id()]); $node = $storage->load($node->id()); + static::assertEquals('Egypt, Hamas exchange fire on Gaza frontier, 1 dead - Reuters', $node->getTitle()); static::assertNotEquals($update_date, $node->changed->value); } @@ -112,7 +119,7 @@ class UpdateNonExistentTest extends FeedsKernelTestBase { * feed get deleted when the 'update_non_existent' setting is set to * '_delete'. */ - public function testDeleteNonExistentItems() { + public function _testDeleteNonExistentItems() { // Set 'update_non_existent' setting to 'unpublish'. $config = $this->feedType->getProcessor()->getConfiguration(); $config['update_non_existent'] = ProcessorInterface::DELETE_NON_EXISTENT;