diff --git a/core/modules/migrate/src/Event/MigrateEvents.php b/core/modules/migrate/src/Event/MigrateEvents.php index e2de42e..7263777 100644 --- a/core/modules/migrate/src/Event/MigrateEvents.php +++ b/core/modules/migrate/src/Event/MigrateEvents.php @@ -47,6 +47,37 @@ const MAP_DELETE = 'migrate.map_delete'; /** + * Name of the event fired when removing a list of entries from a map. + * + * This event allows modules to perform an action whenever a set of rows are + * deleted from a migration's map table (implying they were been rolled back). + * The event listener method receives a MigrateMapDeleteMultipleEvent + * instance. + * + * @Event + * + * @see \Drupal\migrate\Event\MigrateMapDeleteMultipleEvent + * + * @var string + */ + const MAP_DELETE_MULTIPLE = 'migrate.map_delete_multiple'; + + /** + * Name of the event fired when removing all entries from a map. + * + * This event allows modules to perform an action when all rows are deleted + * from a migration's map table (implying they were been rolled back). The + * event listener method receives a MigrateMapDeleteAllEvent instance. + * + * @Event + * + * @see \Drupal\migrate\Event\MigrateMapDeleteAllEvent + * + * @var string + */ + const MAP_DELETE_ALL = 'migrate.map_delete_all'; + + /** * Name of the event fired when beginning a migration import operation. * * This event allows modules to perform an action whenever a migration import @@ -167,6 +198,66 @@ const POST_ROW_DELETE = 'migrate.post_row_delete'; /** + * Name of the event fired when about to delete multiple destination items. + * + * This event allows modules to perform an action whenever a set of items are + * about to be deleted by the destination plugin. The event listener method + * receives a MigrateRowDeleteMultipleEvent instance. + * + * @Event + * + * @see \Drupal\migrate\Event\MigrateRowDeleteMultipleEvent + * + * @var string + */ + const PRE_ROW_DELETE_MULTIPLE = 'migrate.pre_row_delete_multiple'; + + /** + * Name of the event fired just after multiple items were deleted. + * + * This event allows modules to perform an action whenever a set of items were + * deleted by the destination plugin. The event listener method receives a + * MigrateRowDeleteMultipleEvent instance. + * + * @Event + * + * @see \Drupal\migrate\Event\MigrateRowDeleteMultipleEvent + * + * @var string + */ + const POST_ROW_DELETE_MULTIPLE = 'migrate.post_row_delete_multiple'; + + /** + * Name of the event fired when about to delete all destination items. + * + * This event allows modules to perform an action when all items are about to + * be deleted by the destination plugin. The event listener method receives a + * MigrateRowDeleteAllEvent instance. + * + * @Event + * + * @see \Drupal\migrate\Event\MigrateRowDeleteAllEvent + * + * @var string + */ + const PRE_ROW_DELETE_ALL = 'migrate.pre_row_delete_all'; + + /** + * Name of the event fired just after multiple items were deleted. + * + * This event allows modules to perform an action after all items were + * deleted by the destination plugin. The event listener method receives a + * MigrateRowDeleteAllEvent instance. + * + * @Event + * + * @see \Drupal\migrate\Event\MigrateRowDeleteAllEvent + * + * @var string + */ + const POST_ROW_DELETE_ALL = 'migrate.post_row_delete_all'; + + /** * Name of the event fired when saving a message to the idmap. * * This event allows modules to perform an action whenever a message is being diff --git a/core/modules/migrate/src/Event/MigrateMapDeleteAllEvent.php b/core/modules/migrate/src/Event/MigrateMapDeleteAllEvent.php new file mode 100644 index 0000000..535d562 --- /dev/null +++ b/core/modules/migrate/src/Event/MigrateMapDeleteAllEvent.php @@ -0,0 +1,40 @@ +map = $map; + } + + /** + * Gets the map plugin. + * + * @return \Drupal\migrate\Plugin\MigrateIdMapInterface + * The map plugin that caused the event to fire. + */ + public function getMap() { + return $this->map; + } + +} diff --git a/core/modules/migrate/src/Event/MigrateMapDeleteMultipleEvent.php b/core/modules/migrate/src/Event/MigrateMapDeleteMultipleEvent.php new file mode 100644 index 0000000..8e8cb71 --- /dev/null +++ b/core/modules/migrate/src/Event/MigrateMapDeleteMultipleEvent.php @@ -0,0 +1,62 @@ +map = $map; + $this->sourceIds = $source_ids; + } + + /** + * Gets the map plugin. + * + * @return \Drupal\migrate\Plugin\MigrateIdMapInterface + * The map plugin that caused the event to fire. + */ + public function getMap() { + return $this->map; + } + + /** + * Gets the list of source IDs of the items being removed from the map. + * + * @return string[] + * Array of source ID hashes. + */ + public function getSourceIds() { + return $this->sourceIds; + } + +} diff --git a/core/modules/migrate/src/Event/MigrateRowDeleteAllEvent.php b/core/modules/migrate/src/Event/MigrateRowDeleteAllEvent.php new file mode 100644 index 0000000..00517e3 --- /dev/null +++ b/core/modules/migrate/src/Event/MigrateRowDeleteAllEvent.php @@ -0,0 +1,40 @@ +migration = $migration; + } + + /** + * Gets the migration entity. + * + * @return \Drupal\migrate\Plugin\MigrationInterface + * The migration being rolled back. + */ + public function getMigration() { + return $this->migration; + } + +} diff --git a/core/modules/migrate/src/Event/MigrateRowDeleteMultipleEvent.php b/core/modules/migrate/src/Event/MigrateRowDeleteMultipleEvent.php new file mode 100644 index 0000000..c379b51 --- /dev/null +++ b/core/modules/migrate/src/Event/MigrateRowDeleteMultipleEvent.php @@ -0,0 +1,60 @@ +migration = $migration; + $this->destinationIds = $destination_ids; + } + + /** + * Gets the migration entity. + * + * @return \Drupal\migrate\Plugin\MigrationInterface + * The migration being rolled back. + */ + public function getMigration() { + return $this->migration; + } + + /** + * Gets the list of destination IDs. + * + * @return array[] + * The list of destination IDs. + */ + public function getDestinationIds() { + return $this->destinationIds; + } + +} diff --git a/core/modules/migrate/src/MigrateExecutable.php b/core/modules/migrate/src/MigrateExecutable.php index 73e7e00..c30f800 100644 --- a/core/modules/migrate/src/MigrateExecutable.php +++ b/core/modules/migrate/src/MigrateExecutable.php @@ -9,10 +9,13 @@ use Drupal\migrate\Event\MigratePostRowSaveEvent; use Drupal\migrate\Event\MigratePreRowSaveEvent; use Drupal\migrate\Event\MigrateRollbackEvent; +use Drupal\migrate\Event\MigrateRowDeleteAllEvent; use Drupal\migrate\Event\MigrateRowDeleteEvent; +use Drupal\migrate\Event\MigrateRowDeleteMultipleEvent; use Drupal\migrate\Exception\RequirementsException; use Drupal\migrate\Plugin\MigrateIdMapInterface; use Drupal\migrate\Plugin\MigrationInterface; +use Drupal\migrate\Plugin\MigrateDestinationFastRollbackInterface; use Symfony\Component\EventDispatcher\EventDispatcherInterface; /** @@ -314,39 +317,89 @@ public function rollback() { $this->migration->setStatus(MigrationInterface::STATUS_ROLLING_BACK); $id_map = $this->migration->getIdMap(); $destination = $this->migration->getDestinationPlugin(); + $config = $this->migration->getDestinationConfiguration(); + + // A fast rollback will run if all of the following conditions are met: + // - The destination plugin class implements the + // MigrateDestinationFastRollbackInterface interface. + // - The destination plugin is instantiated with the configuration option + // 'fast_rollback' set to TRUE. + // - The batch size is 0 or greater than 1. + if ($destination instanceof MigrateDestinationFastRollbackInterface && !empty($config['fast_rollback'])) { + $batch_size = array_key_exists('fast_rollback_batch_size', $config) ? $config['fast_rollback_batch_size'] : MigrateDestinationFastRollbackInterface::BATCH_SIZE; + } + else { + $batch_size = 1; + } - // Loop through each row in the map, and try to roll it back. - foreach ($id_map as $map_row) { - $destination_key = $id_map->currentDestination(); - if ($destination_key) { - $map_row = $id_map->getRowByDestination($destination_key); - if ($map_row['rollback_action'] == MigrateIdMapInterface::ROLLBACK_DELETE) { - $this->getEventDispatcher() - ->dispatch(MigrateEvents::PRE_ROW_DELETE, new MigrateRowDeleteEvent($this->migration, $destination_key)); - $destination->rollback($destination_key); - $this->getEventDispatcher() - ->dispatch(MigrateEvents::POST_ROW_DELETE, new MigrateRowDeleteEvent($this->migration, $destination_key)); - } - // We're now done with this row, so remove it from the map. - $id_map->deleteDestination($destination_key); - } - else { - // If there is no destination key the import probably failed and we can - // remove the row without further action. + // The destination configuration has requested an "all-in-one" rollback. + if ($batch_size === 0) { + $this->getEventDispatcher()->dispatch(MigrateEvents::PRE_ROW_DELETE_ALL, new MigrateRowDeleteAllEvent($this->migration)); + $destination->rollbackAll(); + $this->getEventDispatcher()->dispatch(MigrateEvents::POST_ROW_DELETE_ALL, new MigrateRowDeleteAllEvent($this->migration)); + + // Clean up the map tables. + $id_map->deleteAll(); + + $return = $this->checkStatus(); + } + // Perform a batch rollback. The per-item rollback is a particular case of + // the batch rollback (the batch size is 1). + else { + $batch_count = 0; + $destination_ids = $source_ids = []; + $id_map->rewind(); + while ($id_map->valid()) { + $batch_count++; + $destination_key = $id_map->currentDestination(); $source_key = $id_map->currentSource(); - $id_map->delete($source_key); - } + $map_row = $id_map->getRowBySource($source_key); + $source_ids[] = $source_key; + if ($destination_key && ($map_row['rollback_action'] == MigrateIdMapInterface::ROLLBACK_DELETE)) { + $destination_ids[] = $destination_key; + } - // Check for memory exhaustion. - if (($return = $this->checkStatus()) != MigrationInterface::RESULT_COMPLETED) { - break; - } + // Advance to next map row. + $id_map->next(); + + if ($batch_count === $batch_size || !$id_map->valid()) { + // Single rollback. + if ($batch_size === 1) { + $destination_key = reset($destination_ids); + $source_key = reset($source_ids); + // Remove the destination item if we have a valid ID. + if ($destination_key) { + $this->getEventDispatcher()->dispatch(MigrateEvents::PRE_ROW_DELETE, new MigrateRowDeleteEvent($this->migration, $destination_key)); + $destination->rollback($destination_key); + $this->getEventDispatcher()->dispatch(MigrateEvents::POST_ROW_DELETE, new MigrateRowDeleteEvent($this->migration, $destination_key)); + } + // Clean up the map tables. + $id_map->delete($source_key); + } + // Multiple rollback. + else { + // Remove destination objects marked for removal. + $this->getEventDispatcher()->dispatch(MigrateEvents::PRE_ROW_DELETE_MULTIPLE, new MigrateRowDeleteMultipleEvent($this->migration, $destination_ids)); + $destination->rollbackMultiple($destination_ids); + $this->getEventDispatcher()->dispatch(MigrateEvents::POST_ROW_DELETE_MULTIPLE, new MigrateRowDeleteMultipleEvent($this->migration, $destination_ids)); + // Clean up the map tables. + $id_map->deleteMultiple($source_ids); + } + $batch_count = 0; + $destination_ids = $source_ids = []; - // If anyone has requested we stop, return the requested result. - if ($this->migration->getStatus() == MigrationInterface::STATUS_STOPPING) { - $return = $this->migration->getInterruptionResult(); - $this->migration->clearInterruptionResult(); - break; + // Check for memory exhaustion. + if (($return = $this->checkStatus()) != MigrationInterface::RESULT_COMPLETED) { + break; + } + + // If anyone has requested we stop, return the requested result. + if ($this->migration->getStatus() == MigrationInterface::STATUS_STOPPING) { + $return = $this->migration->getInterruptionResult(); + $this->migration->clearInterruptionResult(); + break; + } + } } } diff --git a/core/modules/migrate/src/Plugin/MigrateDestinationFastRollbackInterface.php b/core/modules/migrate/src/Plugin/MigrateDestinationFastRollbackInterface.php new file mode 100644 index 0000000..3a6e8dc --- /dev/null +++ b/core/modules/migrate/src/Plugin/MigrateDestinationFastRollbackInterface.php @@ -0,0 +1,79 @@ +deleteMultiple([$source_id_values], $messages_only); + } + + /** + * {@inheritdoc} + */ + public function deleteMultiple(array $source_ids, $messages_only = FALSE) { + // Build a list of source hashes. + if (!$source_ids_hashes = array_map([$this, 'getSourceIDsHash'], $source_ids)) { + return; + } if (!$messages_only) { $map_query = $this->getDatabase()->delete($this->mapTableName()); - $map_query->condition(static::SOURCE_IDS_HASH, $this->getSourceIDsHash($source_id_values)); - // Notify anyone listening of the map row we're about to delete. - $this->eventDispatcher->dispatch(MigrateEvents::MAP_DELETE, new MigrateMapDeleteEvent($this, $source_id_values)); + $map_query->condition(static::SOURCE_IDS_HASH, $source_ids_hashes, 'IN'); + // Notify anyone listening of the map rows we're about to delete. + if (count($source_ids) === 1) { + $this->eventDispatcher->dispatch(MigrateEvents::MAP_DELETE, new MigrateMapDeleteEvent($this, $source_ids[0])); + } + else { + $this->eventDispatcher->dispatch(MigrateEvents::MAP_DELETE_MULTIPLE, new MigrateMapDeleteMultipleEvent($this, $source_ids)); + } $map_query->execute(); } $message_query = $this->getDatabase()->delete($this->messageTableName()); - $message_query->condition(static::SOURCE_IDS_HASH, $this->getSourceIDsHash($source_id_values)); + $message_query->condition(static::SOURCE_IDS_HASH, $source_ids_hashes, 'IN'); $message_query->execute(); } /** * {@inheritdoc} */ + public function deleteAll($messages_only = FALSE) { + if (!$messages_only) { + $this->eventDispatcher->dispatch(MigrateEvents::MAP_DELETE_ALL, new MigrateMapDeleteAllEvent($this)); + $this->getDatabase()->truncate($this->mapTableName())->execute(); + } + $this->getDatabase()->truncate($this->messageTableName())->execute(); + } + + /** + * {@inheritdoc} + */ public function deleteDestination(array $destination_id_values) { $map_query = $this->getDatabase()->delete($this->mapTableName()); $message_query = $this->getDatabase()->delete($this->messageTableName()); diff --git a/core/modules/migrate/tests/modules/migrate_fast_rollback_test/migrate_fast_rollback_test.info.yml b/core/modules/migrate/tests/modules/migrate_fast_rollback_test/migrate_fast_rollback_test.info.yml new file mode 100644 index 0000000..6a70c9f --- /dev/null +++ b/core/modules/migrate/tests/modules/migrate_fast_rollback_test/migrate_fast_rollback_test.info.yml @@ -0,0 +1,6 @@ +name: 'Migrate fast rollback test' +type: module +description: 'Support module for fast rollback destination test.' +package: Testing +version: VERSION +core: 8.x diff --git a/core/modules/migrate/tests/modules/migrate_fast_rollback_test/migrate_fast_rollback_test.install b/core/modules/migrate/tests/modules/migrate_fast_rollback_test/migrate_fast_rollback_test.install new file mode 100644 index 0000000..9457689 --- /dev/null +++ b/core/modules/migrate/tests/modules/migrate_fast_rollback_test/migrate_fast_rollback_test.install @@ -0,0 +1,28 @@ + [ + 'fields' => [ + 'id' => [ + 'type' => 'int', + 'not null' => TRUE, + 'unsigned' => TRUE, + ], + 'name' => [ + 'type' => 'varchar', + 'length' => 255, + ], + ], + 'primary key' => ['id'], + ], + ]; +} diff --git a/core/modules/migrate/tests/modules/migrate_fast_rollback_test/src/Plugin/migrate/destination/TestFastRollbackDestination.php b/core/modules/migrate/tests/modules/migrate_fast_rollback_test/src/Plugin/migrate/destination/TestFastRollbackDestination.php new file mode 100644 index 0000000..c0e73eb --- /dev/null +++ b/core/modules/migrate/tests/modules/migrate_fast_rollback_test/src/Plugin/migrate/destination/TestFastRollbackDestination.php @@ -0,0 +1,110 @@ +setRollbackMethod('item'); + + parent::rollback($destination_identifier); + \Drupal::database()->delete('migrate_fast_rollback_test') + ->condition('id', $destination_identifier['id']) + ->execute(); + } + + /** + * {@inheritdoc} + */ + public function rollbackMultiple(array $destination_ids) { + $this->setRollbackMethod('batch'); + + $ids = array_map(function (array $id_values) { + return $id_values['id']; + }, $destination_ids); + + \Drupal::database()->delete('migrate_fast_rollback_test') + ->condition('id', $ids, 'IN') + ->execute(); + } + + /** + * {@inheritdoc} + */ + public function rollbackAll() { + $this->setRollbackMethod('all'); + \Drupal::database()->truncate('migrate_fast_rollback_test')->execute(); + } + + /** + * {@inheritdoc} + */ + public function import(Row $row, array $old_destination_id_values = []) { + $connection = Database::getConnection(); + $values = $row->getDestination(); + $id = $values['id']; + try { + if (empty($old_destination_id_values)) { + $connection->insert('migrate_fast_rollback_test') + ->fields(array_keys($this->fields())) + ->values($values) + ->execute(); + } + else { + unset($values['id']); + $connection->update('migrate_fast_rollback_test') + ->fields($values) + ->condition('id', $id) + ->execute(); + } + return [$id]; + } + catch (\Exception $exception) { + throw new MigrateException($exception->getMessage()); + } + } + + /** + * {@inheritdoc} + */ + public function getIds() { + return ['id' => ['type' => 'integer']]; + } + + /** + * {@inheritdoc} + */ + public function fields(MigrationInterface $migration = NULL) { + return ['id' => 'ID', 'name' => 'Name']; + } + + /** + * Sets the rollback method type and logs the method calls count. + * + * @param string $rollback_method + * The rollback method: 'item', 'batch' or 'all'. + */ + protected function setRollbackMethod($rollback_method) { + $state = \Drupal::state(); + $default = ['type' => $rollback_method, 'count' => 0]; + $method = $state->get('migrate_fast_rollback_test', $default); + $method['count']++; + $state->set('migrate_fast_rollback_test', $method); + } + +} diff --git a/core/modules/migrate/tests/src/Kernel/MigrateFastRollbackTest.php b/core/modules/migrate/tests/src/Kernel/MigrateFastRollbackTest.php new file mode 100644 index 0000000..74a1153 --- /dev/null +++ b/core/modules/migrate/tests/src/Kernel/MigrateFastRollbackTest.php @@ -0,0 +1,170 @@ + [], + 'method' => 'item', + 'calls' => static::COUNT, + ], + // Batch with default batch size. + [ + 'config' => [ + 'fast_rollback' => TRUE, + ], + 'method' => 'batch', + 'calls' => ceil(static::COUNT / 50), + ], + // Batch with a custom batch size. + [ + 'config' => [ + 'fast_rollback' => TRUE, + 'fast_rollback_batch_size' => 100, + ], + 'method' => 'batch', + 'calls' => ceil(static::COUNT / 100), + ], + // Batch with size 1. This should fall back to a per-item rollback. + [ + 'config' => [ + 'fast_rollback' => TRUE, + 'fast_rollback_batch_size' => 1, + ], + 'method' => 'item', + 'calls' => static::COUNT, + ], + // All on one. + [ + 'config' => [ + 'fast_rollback' => TRUE, + 'fast_rollback_batch_size' => 0, + ], + 'method' => 'all', + 'calls' => 1, + ], + ]; + } + + /** + * Tests migration fast rollback destinations. + * + * @dataProvider providerFastRollback + * @covers ::rollbackMultiple + * @covers ::rollbackAll + * + * @param array $destination_settings + * Settings to be configured with the destination plugin. + * @param string $rollback_method + * The used rollback method: 'item', 'batch', 'all'. + * @param int $calls + * The expected number of times the rollback method is called. + */ + public function testFastRollback(array $destination_settings, $rollback_method, $calls) { + $this->installSchema('migrate_fast_rollback_test', ['migrate_fast_rollback_test']); + $this->installSchema('system', ['key_value']); + + $definition = $this->getMigration($destination_settings); + $migration = \Drupal::service('plugin.manager.migration')->createStubMigration($definition); + + $executable = new MigrateExecutable($migration, new MigrateMessage()); + /** @var \Drupal\migrate\Plugin\MigrateIdMapInterface $id_map_plugin */ + $id_map_plugin = $migration->getIdMap(); + + // Import into custom table. + $result = $executable->import(); + + // Check that the import was successfully. + $this->assertEquals(MigrationInterface::RESULT_COMPLETED, $result); + $this->assertEquals(static::COUNT, $id_map_plugin->importedCount()); + + // Rollback. + $result = $executable->rollback(); + + // Load the state value stored by the rollback method. + $rollback = \Drupal::state()->get('migrate_fast_rollback_test'); + + // Check that the rollback was successfully. + $this->assertEquals(MigrationInterface::RESULT_COMPLETED, $result); + $count = (int) Database::getConnection()->select('migrate_fast_rollback_test') + ->countQuery()->execute()->fetchField(); + $this->assertEquals(0, $count); + + // Check that the appropriate rollback method has been used. + $this->assertEquals($rollback_method, $rollback['type']); + + // Check that the rollback method was called the expected number of times. + $this->assertEquals($calls, $rollback['count']); + } + + /** + * Builds an embedded source migration. + * + * @param array $destination_settings + * Settings to be configured with the destination plugin. + * + * @return array + * The migration definition. + */ + protected function getMigration(array $destination_settings) { + $destination = ['plugin' => 'test_fast_rollback'] + $destination_settings; + + $data_rows = []; + foreach (range(1, static::COUNT) as $item) { + $data_rows[] = ['id' => $item, 'name' => $this->randomString()]; + } + + return [ + 'source' => [ + 'plugin' => 'embedded_data', + 'data_rows' => $data_rows, + 'ids' => ['id' => ['type' => 'integer']], + ], + 'process' => [ + 'id' => 'id', + 'name' => 'name', + ], + 'destination' => $destination, + ]; + } + +}