diff --git a/core/modules/aggregator/migration_templates/d6_aggregator_feed.yml b/core/modules/aggregator/migration_templates/d6_aggregator_feed.yml index cad1553..105d09b 100644 --- a/core/modules/aggregator/migration_templates/d6_aggregator_feed.yml +++ b/core/modules/aggregator/migration_templates/d6_aggregator_feed.yml @@ -2,6 +2,7 @@ id: d6_aggregator_feed label: Aggregator feeds migration_tags: - Drupal 6 +audit_ids: true source: plugin: aggregator_feed process: diff --git a/core/modules/aggregator/migration_templates/d6_aggregator_item.yml b/core/modules/aggregator/migration_templates/d6_aggregator_item.yml index e14dbd6..022a780 100644 --- a/core/modules/aggregator/migration_templates/d6_aggregator_item.yml +++ b/core/modules/aggregator/migration_templates/d6_aggregator_item.yml @@ -2,6 +2,7 @@ id: d6_aggregator_item label: Aggregator items migration_tags: - Drupal 6 +audit_ids: true source: plugin: aggregator_item process: diff --git a/core/modules/aggregator/migration_templates/d7_aggregator_feed.yml b/core/modules/aggregator/migration_templates/d7_aggregator_feed.yml index 5dbeb25..9a038b7 100644 --- a/core/modules/aggregator/migration_templates/d7_aggregator_feed.yml +++ b/core/modules/aggregator/migration_templates/d7_aggregator_feed.yml @@ -2,6 +2,7 @@ id: d7_aggregator_feed label: Aggregator feeds migration_tags: - Drupal 7 +audit_ids: true source: plugin: aggregator_feed process: diff --git a/core/modules/aggregator/migration_templates/d7_aggregator_item.yml b/core/modules/aggregator/migration_templates/d7_aggregator_item.yml index 054ba43..ac76a8c 100644 --- a/core/modules/aggregator/migration_templates/d7_aggregator_item.yml +++ b/core/modules/aggregator/migration_templates/d7_aggregator_item.yml @@ -2,6 +2,7 @@ id: d7_aggregator_item label: Aggregator items migration_tags: - Drupal 7 +audit_ids: true source: plugin: aggregator_item process: diff --git a/core/modules/block_content/migration_templates/d6_custom_block.yml b/core/modules/block_content/migration_templates/d6_custom_block.yml index 55fbcb5..d2826c7 100644 --- a/core/modules/block_content/migration_templates/d6_custom_block.yml +++ b/core/modules/block_content/migration_templates/d6_custom_block.yml @@ -2,6 +2,7 @@ id: d6_custom_block label: Custom blocks migration_tags: - Drupal 6 +audit_ids: true source: plugin: d6_box process: diff --git a/core/modules/block_content/migration_templates/d7_custom_block.yml b/core/modules/block_content/migration_templates/d7_custom_block.yml index ca06cf0..1089ca3 100644 --- a/core/modules/block_content/migration_templates/d7_custom_block.yml +++ b/core/modules/block_content/migration_templates/d7_custom_block.yml @@ -2,6 +2,7 @@ id: d7_custom_block label: Custom blocks migration_tags: - Drupal 7 +audit_ids: true source: plugin: d7_block_custom process: diff --git a/core/modules/comment/migration_templates/d6_comment.yml b/core/modules/comment/migration_templates/d6_comment.yml index 06820d4..4add74b 100644 --- a/core/modules/comment/migration_templates/d6_comment.yml +++ b/core/modules/comment/migration_templates/d6_comment.yml @@ -2,6 +2,7 @@ id: d6_comment label: Comments migration_tags: - Drupal 6 +audit_ids: true source: plugin: d6_comment constants: diff --git a/core/modules/comment/migration_templates/d7_comment.yml b/core/modules/comment/migration_templates/d7_comment.yml index 94a2884..c020ab2 100644 --- a/core/modules/comment/migration_templates/d7_comment.yml +++ b/core/modules/comment/migration_templates/d7_comment.yml @@ -2,6 +2,7 @@ id: d7_comment label: Comments migration_tags: - Drupal 7 +audit_ids: true source: plugin: d7_comment constants: diff --git a/core/modules/file/migration_templates/d6_file.yml b/core/modules/file/migration_templates/d6_file.yml index 8371d45..8589978 100644 --- a/core/modules/file/migration_templates/d6_file.yml +++ b/core/modules/file/migration_templates/d6_file.yml @@ -4,6 +4,7 @@ id: d6_file label: Files migration_tags: - Drupal 6 +audit_ids: true source: plugin: d6_file constants: diff --git a/core/modules/file/migration_templates/d7_file.yml b/core/modules/file/migration_templates/d7_file.yml index 3fee046..8bc2147 100644 --- a/core/modules/file/migration_templates/d7_file.yml +++ b/core/modules/file/migration_templates/d7_file.yml @@ -4,6 +4,7 @@ id: d7_file label: Files migration_tags: - Drupal 7 +audit_ids: true source: plugin: d7_file scheme: public diff --git a/core/modules/file/migration_templates/d7_file_private.yml b/core/modules/file/migration_templates/d7_file_private.yml index 9c6b8e2..d2e7c89 100644 --- a/core/modules/file/migration_templates/d7_file_private.yml +++ b/core/modules/file/migration_templates/d7_file_private.yml @@ -2,6 +2,7 @@ id: d7_file_private label: Files migration_tags: - Drupal 7 +audit_ids: true source: plugin: d7_file scheme: private diff --git a/core/modules/menu_link_content/migration_templates/d6_menu_links.yml b/core/modules/menu_link_content/migration_templates/d6_menu_links.yml index 2c8ad4a..2a79d35 100644 --- a/core/modules/menu_link_content/migration_templates/d6_menu_links.yml +++ b/core/modules/menu_link_content/migration_templates/d6_menu_links.yml @@ -2,6 +2,7 @@ id: d6_menu_links label: Menu links migration_tags: - Drupal 6 +audit_ids: true source: plugin: menu_link process: diff --git a/core/modules/menu_link_content/migration_templates/d7_menu_links.yml b/core/modules/menu_link_content/migration_templates/d7_menu_links.yml index 200a792..854a056 100644 --- a/core/modules/menu_link_content/migration_templates/d7_menu_links.yml +++ b/core/modules/menu_link_content/migration_templates/d7_menu_links.yml @@ -2,6 +2,7 @@ id: d7_menu_links label: Menu links migration_tags: - Drupal 7 +audit_ids: true source: plugin: menu_link constants: diff --git a/core/modules/migrate/migrate.services.yml b/core/modules/migrate/migrate.services.yml index 1a4f64d..1b95191 100644 --- a/core/modules/migrate/migrate.services.yml +++ b/core/modules/migrate/migrate.services.yml @@ -30,3 +30,5 @@ services: plugin.manager.migration: class: Drupal\migrate\Plugin\MigrationPluginManager arguments: ['@module_handler', '@cache.discovery_migration', '@language_manager'] + migrate.id_auditor: + class: Drupal\migrate\MigrateIdAuditor diff --git a/core/modules/migrate/src/MigrateIdAuditor.php b/core/modules/migrate/src/MigrateIdAuditor.php new file mode 100644 index 0000000..c350df1 --- /dev/null +++ b/core/modules/migrate/src/MigrateIdAuditor.php @@ -0,0 +1,60 @@ +getPluginDefinition(); + if (!empty($definition['audit_ids'])) { + $destination = $migration->getDestinationPlugin(); + $id_map = $migration->getIdMap(); + if ($destination instanceof MigrateDestinationAuditInterface && $id_map instanceof MigrateIdMapAuditInterface) { + $field_name = $destination->getAuditedIdFieldName(); + if ($destination->getHighestDestinationId($field_name) > $id_map->getHighestMigratedId($field_name)) { + $conflicts += $this->buildConflict($migration); + } + } + } + } + return $conflicts; + } + + /** + * Build conflict label. + * + * @param \Drupal\migrate\Plugin\MigrationInterface $migration + * + * @return array + */ + protected function buildConflict(MigrationInterface $migration) { + $conflict = []; + $base_id = $migration->getBaseId(); + $label = $migration->label(); + if (is_string($label)) { + $conflict[$base_id] = $label; + } + elseif ($label instanceof TranslatableMarkup) { + $conflict[$base_id] = $label->render(); + if (isset($label->getArguments()['@label'])) { + $conflict[$base_id] = $label->getArguments()['@label']; + } + } + + return $conflict; + } + +} diff --git a/core/modules/migrate/src/MigrateIdAuditorInterface.php b/core/modules/migrate/src/MigrateIdAuditorInterface.php new file mode 100644 index 0000000..036c884 --- /dev/null +++ b/core/modules/migrate/src/MigrateIdAuditorInterface.php @@ -0,0 +1,28 @@ +configuration['translations']); } @@ -289,4 +294,22 @@ protected function getDefinitionFromEntity($key) { ] + $field_definition->getSettings(); } + /** + * {@inheritdoc} + */ + public function getAuditedIdFieldName() { + return $this->getKey('id'); + } + + /** + * {@inheritdoc} + */ + public function getHighestDestinationId($field_name) { + $query = $this->storage->getQuery() + ->sort($field_name) + ->range(0, 1); + $found = $query->execute(); + return (int) reset($found); + } + } diff --git a/core/modules/migrate/src/Plugin/migrate/destination/EntityRevision.php b/core/modules/migrate/src/Plugin/migrate/destination/EntityRevision.php index b0db476..178b867 100644 --- a/core/modules/migrate/src/Plugin/migrate/destination/EntityRevision.php +++ b/core/modules/migrate/src/Plugin/migrate/destination/EntityRevision.php @@ -78,4 +78,11 @@ public function getIds() { throw new MigrateException('This entity type does not support revisions.'); } + /** + * {@inheritdoc} + */ + public function getAuditedIdFieldName() { + return $this->getKey('revision'); + } + } diff --git a/core/modules/migrate/src/Plugin/migrate/id_map/Sql.php b/core/modules/migrate/src/Plugin/migrate/id_map/Sql.php index 6bdd51e..dadcdec 100644 --- a/core/modules/migrate/src/Plugin/migrate/id_map/Sql.php +++ b/core/modules/migrate/src/Plugin/migrate/id_map/Sql.php @@ -7,11 +7,13 @@ use Drupal\Core\Plugin\ContainerFactoryPluginInterface; use Drupal\Core\Plugin\PluginBase; use Drupal\migrate\MigrateMessage; +use Drupal\migrate\Plugin\MigrateIdMapAuditInterface; use Drupal\migrate\Plugin\MigrationInterface; use Drupal\migrate\Event\MigrateIdMapMessageEvent; use Drupal\migrate\MigrateException; use Drupal\migrate\MigrateMessageInterface; use Drupal\migrate\Plugin\MigrateIdMapInterface; +use Drupal\migrate\Plugin\MigrationPluginManagerInterface; use Drupal\migrate\Row; use Drupal\migrate\Event\MigrateEvents; use Drupal\migrate\Event\MigrateMapSaveEvent; @@ -27,7 +29,7 @@ * * @PluginID("sql") */ -class Sql extends PluginBase implements MigrateIdMapInterface, ContainerFactoryPluginInterface { +class Sql extends PluginBase implements MigrateIdMapInterface, ContainerFactoryPluginInterface, MigrateIdMapAuditInterface { /** * Column name of hashed source id values. @@ -42,6 +44,13 @@ class Sql extends PluginBase implements MigrateIdMapInterface, ContainerFactoryP protected $eventDispatcher; /** + * The migration plugin manager to have access to the list of migrations. + * + * @var \Drupal\migrate\Plugin\MigrationPluginManagerInterface + */ + protected $migrationPluginManager; + + /** * The migration map table name. * * @var string @@ -152,11 +161,16 @@ class Sql extends PluginBase implements MigrateIdMapInterface, ContainerFactoryP * The configuration for the plugin. * @param \Drupal\migrate\Plugin\MigrationInterface $migration * The migration to do. + * @param \Symfony\Component\EventDispatcher\EventDispatcherInterface $event_dispatcher + * The event dispatcher. + * @param \Drupal\migrate\Plugin\MigrationPluginManagerInterface $migration_plugin_manager + * The migration plugin manager. */ - public function __construct(array $configuration, $plugin_id, $plugin_definition, MigrationInterface $migration, EventDispatcherInterface $event_dispatcher) { + public function __construct(array $configuration, $plugin_id, $plugin_definition, MigrationInterface $migration, EventDispatcherInterface $event_dispatcher, MigrationPluginManagerInterface $migration_plugin_manager) { parent::__construct($configuration, $plugin_id, $plugin_definition); $this->migration = $migration; $this->eventDispatcher = $event_dispatcher; + $this->migrationPluginManager = $migration_plugin_manager; $this->message = new MigrateMessage(); } @@ -169,7 +183,8 @@ public static function create(ContainerInterface $container, array $configuratio $plugin_id, $plugin_definition, $migration, - $container->get('event_dispatcher') + $container->get('event_dispatcher'), + $container->get('plugin.manager.migration') ); } @@ -925,4 +940,48 @@ public function valid() { return $this->currentRow !== FALSE; } + /** + * {@inheritdoc} + */ + public function getHighestMigratedId($field_name) { + $sql_field = $this->destinationIdFields()[$field_name]; + $migration_id = $this->migration->id(); + + // List of mapping tables to look in for the highest ID. + $map_tables = [ + $migration_id => $this->mapTableName(), + ]; + + // If there's a bundle, it means we have a derived migration and we need to + // find all the mapping tables from the related derived migrations. + if ($base_id = substr($migration_id, 0, strpos($migration_id, ':'))) { + $migrations = $this->migrationPluginManager->getDefinitions(); + foreach ($migrations as $migration_id => $migration) { + if ($migration['id'] == $base_id) { + // Get this derived migration's mapping table and add it to the list + // of mapping tables to look in for the highest ID. + $stub = $this->migrationPluginManager->createInstance($migration_id); + $map_tables[$migration_id] = $stub->getIdMap()->mapTableName(); + } + } + } + + // Look in all the derived migrations' mapping tables for their highest ID. + $ids = []; + foreach ($map_tables as $map_table) { + if (!$this->getDatabase()->schema()->tableExists($map_table)) { + return 0; + } + + $query = $this->getDatabase()->select($map_table, 'map') + ->fields('map', [$sql_field]) + ->orderBy($sql_field, 'DESC') + ->range(0, 1); + $ids[] = $query->execute()->fetchField(); + } + + // Return the highest of the highest IDs. + return max($ids); + } + } diff --git a/core/modules/migrate/tests/src/Unit/MigrateSqlIdMapEnsureTablesTest.php b/core/modules/migrate/tests/src/Unit/MigrateSqlIdMapEnsureTablesTest.php index 3c3cba8..269728d 100644 --- a/core/modules/migrate/tests/src/Unit/MigrateSqlIdMapEnsureTablesTest.php +++ b/core/modules/migrate/tests/src/Unit/MigrateSqlIdMapEnsureTablesTest.php @@ -232,7 +232,8 @@ protected function runEnsureTablesTest($schema) { ->willReturn($plugin); /** @var \Symfony\Component\EventDispatcher\EventDispatcherInterface $event_dispatcher */ $event_dispatcher = $this->getMock('Symfony\Component\EventDispatcher\EventDispatcherInterface'); - $map = new TestSqlIdMap($database, [], 'sql', [], $migration, $event_dispatcher); + $migration_plugin_manager = $this->getMock('Drupal\migrate\Plugin\MigrationPluginManagerInterface'); + $map = new TestSqlIdMap($database, [], 'sql', [], $migration, $event_dispatcher, $migration_plugin_manager); $map->getDatabase(); } diff --git a/core/modules/migrate/tests/src/Unit/MigrateSqlIdMapTest.php b/core/modules/migrate/tests/src/Unit/MigrateSqlIdMapTest.php index 2ad2b3d..c636b09 100644 --- a/core/modules/migrate/tests/src/Unit/MigrateSqlIdMapTest.php +++ b/core/modules/migrate/tests/src/Unit/MigrateSqlIdMapTest.php @@ -111,8 +111,9 @@ protected function getIdMap() { ->method('getDestinationPlugin') ->willReturn($plugin); $event_dispatcher = $this->getMock('Symfony\Component\EventDispatcher\EventDispatcherInterface'); + $migration_plugin_manager = $this->getMock('Drupal\migrate\Plugin\MigrationPluginManagerInterface'); - $id_map = new TestSqlIdMap($this->database, [], 'sql', [], $migration, $event_dispatcher); + $id_map = new TestSqlIdMap($this->database, [], 'sql', [], $migration, $event_dispatcher, $migration_plugin_manager); $migration ->method('getIdMap') ->willReturn($id_map); diff --git a/core/modules/migrate/tests/src/Unit/TestSqlIdMap.php b/core/modules/migrate/tests/src/Unit/TestSqlIdMap.php index 6d33338..f3ee60a 100644 --- a/core/modules/migrate/tests/src/Unit/TestSqlIdMap.php +++ b/core/modules/migrate/tests/src/Unit/TestSqlIdMap.php @@ -6,6 +6,7 @@ use Drupal\migrate\Plugin\MigrationInterface; use Drupal\migrate\MigrateException; use Drupal\migrate\Plugin\migrate\id_map\Sql; +use Drupal\migrate\Plugin\MigrationPluginManagerInterface; use Symfony\Component\EventDispatcher\EventDispatcherInterface; /** @@ -28,10 +29,12 @@ class TestSqlIdMap extends Sql implements \Iterator { * The migration to do. * @param \Symfony\Component\EventDispatcher\EventDispatcherInterface $event_dispatcher * The event dispatcher service. + * @param \Drupal\migrate\Plugin\MigrationPluginManagerInterface $migration_plugin_manager + * The migration plugin manager. */ - public function __construct(Connection $database, array $configuration, $plugin_id, $plugin_definition, MigrationInterface $migration, EventDispatcherInterface $event_dispatcher) { + public function __construct(Connection $database, array $configuration, $plugin_id, $plugin_definition, MigrationInterface $migration, EventDispatcherInterface $event_dispatcher, MigrationPluginManagerInterface $migration_plugin_manager) { $this->database = $database; - parent::__construct($configuration, $plugin_id, $plugin_definition, $migration, $event_dispatcher); + parent::__construct($configuration, $plugin_id, $plugin_definition, $migration, $event_dispatcher, $migration_plugin_manager); } /** diff --git a/core/modules/migrate_drupal/tests/src/Kernel/d6/MigrateDrupal6AuditIdsTest.php b/core/modules/migrate_drupal/tests/src/Kernel/d6/MigrateDrupal6AuditIdsTest.php new file mode 100644 index 0000000..7a1ae8f --- /dev/null +++ b/core/modules/migrate_drupal/tests/src/Kernel/d6/MigrateDrupal6AuditIdsTest.php @@ -0,0 +1,197 @@ +coreModuleListDataProvider()); + parent::setUp(); + + // Install required schemas. + $this->installEntitySchema('aggregator_item'); + $this->installEntitySchema('aggregator_feed'); + $this->installEntitySchema('block_content'); + $this->installEntitySchema('comment'); + $this->installEntitySchema('file'); + $this->installEntitySchema('menu_link_content'); + $this->installEntitySchema('node'); + $this->installEntitySchema('taxonomy_term'); + $this->installSchema('book', ['book']); + $this->installSchema('dblog', ['watchdog']); + $this->installSchema('forum', ['forum_index']); + $this->installSchema('search', ['search_dataset']); + $this->installSchema('system', ['sequences']); + $this->installSchema('tracker', ['tracker_node', 'tracker_user']); + } + + /** + * Tests multiple migrations to the same destination with no ID conflicts. + */ + public function testMultipleMigrationWithoutIdConflicts() { + // Create a node of type page. + Node::create(['type' => 'page', 'title' => 'foo'])->save(); + + // Insert data in the d6_node:page migration mappping table to simulate a + // previously migrated node. + $table_name = $this->getMigration('d6_node:page')->getIdMap()->mapTableName(); + $this->container->get('database')->insert($table_name) + ->fields([ + 'source_ids_hash' => 1, + 'sourceid1' => 1, + 'destid1' => 1, + ]) + ->execute(); + + // Audit the IDs of the d6_node migrations for the page & article node type. + // There should be no conflicts since the highest destination ID should be + // equal to the highest migrated ID, as found in the aggregated mapping + // tables of the two node migrations. + $conflicts = $this->container->get('migrate.id_auditor')->auditIds([ + $this->getMigration('d6_node:page'), + $this->getMigration('d6_node:article'), + ]); + $this->assertTrue(empty($conflicts)); + } + + /** + * Tests all migrations with no ID conflicts. + */ + public function testAllMigrationsWithNoIdConflicts() { + $plugin_manager = $this->container->get('plugin.manager.migration'); + + // Get all Drupal 6 migrations. + $migrations = $plugin_manager->createInstancesByTag('Drupal 6'); + + // Audit the IDs of all migrations. There should be no conflicts since no + // content has been created. + $conflicts = $this->container->get('migrate.id_auditor')->auditIds($migrations); + $this->assertTrue(empty($conflicts)); + } + + /** + * Tests all migrations with ID conflicts. + */ + public function testAllMigrationsWithIdConflicts() { + $plugin_manager = $this->container->get('plugin.manager.migration'); + + // Get all Drupal 6 migrations. + $migrations = $plugin_manager->createInstancesByTag('Drupal 6'); + + // Create content. + $this->createContent(); + + // Audit the IDs of all migrations. There should be conflicts since content + // has been created. + $conflicts = $this->container->get('migrate.id_auditor')->auditIds($migrations); + ksort($conflicts); + + $expected = [ + 'd6_aggregator_feed', + 'd6_aggregator_item', + 'd6_comment', + 'd6_custom_block', + 'd6_file', + 'd6_menu_links', + 'd6_node', + 'd6_node_revision', + 'd6_taxonomy_term', + 'd6_term_node_revision', + 'd6_user', + ]; + + $this->assertSame($expected, array_keys($conflicts)); + } + + protected function createContent() { + // Create an aggregator feed. + $feed = Feed::create([ + 'title' => 'feed', + 'url' => 'http://www.example.com', + ]); + $feed->save(); + + // Create an aggregator feed item. + $item = Item::create([ + 'title' => 'feed item', + 'fid' => $feed->id(), + 'link' => 'http://www.example.com', + ]); + $item->save(); + + // Create a block content. + $block = BlockContent::create([ + 'info' => 'block', + 'type' => 'block', + ]); + $block->save(); + + // Create a node. + $node = Node::create([ + 'type' => 'page', + 'title' => 'page', + ]); + $node->save(); + + // Create a comment. + $comment = Comment::create([ + 'comment_type' => 'comment', + 'field_name' => 'comment', + 'entity_type' => 'node', + 'entity_id' => $node->id(), + ]); + $comment->save(); + + // Create a file. + $file = File::create([ + 'uri' => 'public://example.txt', + ]); + $file->save(); + + // Create a menu link. + $menu_link = MenuLinkContent::create([ + 'title' => 'menu link', + 'link' => ['uri' => 'http://www.example.com'], + 'menu_name' => 'tools', + ]); + $menu_link->save(); + + // Create a taxonomy term. + $term = Term::create([ + 'name' => 'term', + 'vid' => 'term', + ]); + $term->save(); + + // Create a user. + $user = User::create([ + 'uid' => 2, + 'name' => 'user', + 'mail' => 'user@example.com', + ]); + $user->save(); + } + +} diff --git a/core/modules/migrate_drupal/tests/src/Kernel/d7/MigrateDrupal7AuditIdsTest.php b/core/modules/migrate_drupal/tests/src/Kernel/d7/MigrateDrupal7AuditIdsTest.php new file mode 100644 index 0000000..7c20abc --- /dev/null +++ b/core/modules/migrate_drupal/tests/src/Kernel/d7/MigrateDrupal7AuditIdsTest.php @@ -0,0 +1,197 @@ +coreModuleListDataProvider()); + parent::setUp(); + + // Install required schemas. + $this->installEntitySchema('aggregator_item'); + $this->installEntitySchema('aggregator_feed'); + $this->installEntitySchema('block_content'); + $this->installEntitySchema('comment'); + $this->installEntitySchema('file'); + $this->installEntitySchema('menu_link_content'); + $this->installEntitySchema('node'); + $this->installEntitySchema('taxonomy_term'); + $this->installSchema('book', ['book']); + $this->installSchema('dblog', ['watchdog']); + $this->installSchema('forum', ['forum_index']); + $this->installSchema('search', ['search_dataset']); + $this->installSchema('system', ['sequences']); + $this->installSchema('tracker', ['tracker_node', 'tracker_user']); + } + + /** + * Tests multiple migrations to the same destination with no ID conflicts. + */ + public function testMultipleMigrationWithoutIdConflicts() { + // Create a node of type page. + Node::create(['type' => 'page', 'title' => 'foo'])->save(); + + // Insert data in the d7_node:page migration mappping table to simulate a + // previously migrated node. + $table_name = $this->getMigration('d7_node:page')->getIdMap()->mapTableName(); + $this->container->get('database')->insert($table_name) + ->fields([ + 'source_ids_hash' => 1, + 'sourceid1' => 1, + 'destid1' => 1, + ]) + ->execute(); + + // Audit the IDs of the d7_node migrations for the page & article node type. + // There should be no conflicts since the highest destination ID should be + // equal to the highest migrated ID, as found in the aggregated mapping + // tables of the two node migrations. + $conflicts = $this->container->get('migrate.id_auditor')->auditIds([ + $this->getMigration('d7_node:page'), + $this->getMigration('d7_node:article'), + ]); + $this->assertTrue(empty($conflicts)); + } + + /** + * Tests all migrations with no ID conflicts. + */ + public function testAllMigrationsWithNoIdConflicts() { + $plugin_manager = $this->container->get('plugin.manager.migration'); + + // Get all Drupal 7 migrations. + $migrations = $plugin_manager->createInstancesByTag('Drupal 7'); + + // Audit the IDs of all migrations. There should be no conflicts since no + // content has been created. + $conflicts = $this->container->get('migrate.id_auditor')->auditIds($migrations); + $this->assertTrue(empty($conflicts)); + } + + /** + * Tests all migrations with ID conflicts. + */ + public function testAllMigrationsWithIdConflicts() { + $plugin_manager = $this->container->get('plugin.manager.migration'); + + // Get all Drupal 7 migrations. + $migrations = $plugin_manager->createInstancesByTag('Drupal 7'); + + // Create content. + $this->createContent(); + + // Audit the IDs of all migrations. There should be conflicts since content + // has been created. + $conflicts = $this->container->get('migrate.id_auditor')->auditIds($migrations); + ksort($conflicts); + + $expected = [ + 'd7_aggregator_feed', + 'd7_aggregator_item', + 'd7_comment', + 'd7_custom_block', + 'd7_file', + 'd7_file_private', + 'd7_menu_links', + 'd7_node', + 'd7_node_revision', + 'd7_taxonomy_term', + 'd7_user', + ]; + + $this->assertSame($expected, array_keys($conflicts)); + } + + protected function createContent() { + // Create an aggregator feed. + $feed = Feed::create([ + 'title' => 'feed', + 'url' => 'http://www.example.com', + ]); + $feed->save(); + + // Create an aggregator feed item. + $item = Item::create([ + 'title' => 'feed item', + 'fid' => $feed->id(), + 'link' => 'http://www.example.com', + ]); + $item->save(); + + // Create a block content. + $block = BlockContent::create([ + 'info' => 'block', + 'type' => 'block', + ]); + $block->save(); + + // Create a node. + $node = Node::create([ + 'type' => 'page', + 'title' => 'page', + ]); + $node->save(); + + // Create a comment. + $comment = Comment::create([ + 'comment_type' => 'comment', + 'field_name' => 'comment', + 'entity_type' => 'node', + 'entity_id' => $node->id(), + ]); + $comment->save(); + + // Create a file. + $file = File::create([ + 'uri' => 'public://example.txt', + ]); + $file->save(); + + // Create a menu link. + $menu_link = MenuLinkContent::create([ + 'title' => 'menu link', + 'link' => ['uri' => 'http://www.example.com'], + 'menu_name' => 'tools', + ]); + $menu_link->save(); + + // Create a taxonomy term. + $term = Term::create([ + 'name' => 'term', + 'vid' => 'term', + ]); + $term->save(); + + // Create a user. + $user = User::create([ + 'uid' => 2, + 'name' => 'user', + 'mail' => 'user@example.com', + ]); + $user->save(); + } + +} diff --git a/core/modules/migrate_drupal_ui/src/Form/MigrateUpgradeForm.php b/core/modules/migrate_drupal_ui/src/Form/MigrateUpgradeForm.php index ffe4313..8a29b5e 100644 --- a/core/modules/migrate_drupal_ui/src/Form/MigrateUpgradeForm.php +++ b/core/modules/migrate_drupal_ui/src/Form/MigrateUpgradeForm.php @@ -7,7 +7,11 @@ use Drupal\Core\Form\FormStateInterface; use Drupal\Core\Render\RendererInterface; use Drupal\Core\State\StateInterface; +use Drupal\Core\StringTranslation\TranslatableMarkup; use Drupal\Core\Url; +use Drupal\migrate\MigrateIdAuditorInterface; +use Drupal\migrate\Plugin\MigrateDestinationAuditInterface; +use Drupal\migrate\Plugin\MigrationInterface; use Drupal\migrate\Plugin\MigrationPluginManagerInterface; use Drupal\migrate_drupal_ui\Batch\MigrateUpgradeImportBatch; use Drupal\migrate_drupal\MigrationConfigurationTrait; @@ -49,6 +53,13 @@ class MigrateUpgradeForm extends ConfirmFormBase { protected $pluginManager; /** + * The ID conflict auditor. + * + * @var \Drupal\migrate\MigrateIdAuditorInterface $idAuditor + */ + protected $idAuditor; + + /** * Constructs the MigrateUpgradeForm. * * @param \Drupal\Core\State\StateInterface $state @@ -59,12 +70,15 @@ class MigrateUpgradeForm extends ConfirmFormBase { * The renderer service. * @param \Drupal\migrate\Plugin\MigrationPluginManagerInterface $plugin_manager * The migration plugin manager. + * @param \Drupal\migrate\MigrateIdAuditorInterface $id_auditor + * The ID conflict auditor. */ - public function __construct(StateInterface $state, DateFormatterInterface $date_formatter, RendererInterface $renderer, MigrationPluginManagerInterface $plugin_manager) { + public function __construct(StateInterface $state, DateFormatterInterface $date_formatter, RendererInterface $renderer, MigrationPluginManagerInterface $plugin_manager, MigrateIdAuditorInterface $id_auditor) { $this->state = $state; $this->dateFormatter = $date_formatter; $this->renderer = $renderer; $this->pluginManager = $plugin_manager; + $this->idAuditor = $id_auditor; } /** @@ -75,7 +89,8 @@ public static function create(ContainerInterface $container) { $container->get('state'), $container->get('date.formatter'), $container->get('renderer'), - $container->get('plugin.manager.migration') + $container->get('plugin.manager.migration'), + $container->get('migrate.id_auditor') ); } @@ -98,6 +113,9 @@ public function buildForm(array $form, FormStateInterface $form_state) { case 'credentials': return $this->buildCredentialForm($form, $form_state); + case 'confirm_id_conflicts': + return $this->buildIdConflictForm($form, $form_state); + case 'confirm': return $this->buildConfirmForm($form, $form_state); @@ -419,6 +437,121 @@ public function validateCredentialForm(array &$form, FormStateInterface $form_st */ public function submitCredentialForm(array &$form, FormStateInterface $form_state) { // Indicate the next step is confirmation. + $form_state->set('step', 'confirm_id_conflicts'); + $form_state->setRebuild(); + } + + /** + * Confirmation form for ID conflicts. + * + * @param array $form + * An associative array containing the structure of the form. + * @param \Drupal\Core\Form\FormStateInterface $form_state + * The current state of the form. + * + * @return array + * The form structure. + */ + public function buildIdConflictForm(array &$form, FormStateInterface $form_state) { + // Check if there are conflicts. If none, just skip this form! + $migration_ids = array_keys($form_state->get('migrations')); + $migrations = $this->pluginManager->createInstances($migration_ids); + $conflicts = $this->idAuditor->auditIds($migrations); + + $i18n = []; + foreach ($migrations as $migration) { + if ($migration->getDestinationPlugin() instanceof MigrateDestinationAuditInterface) { + if ($migration->getDestinationPlugin()->isTranslationDestination()) { + $i18n += $this->buildConflict($migration); + } + } + } + if (empty($conflicts) && empty($i18n)) { + $form_state->set('step', 'confirm'); + return $this->buildForm($form, $form_state); + } + + $form['warning'] = [ + '#type' => 'markup', + '#markup' => '

' . $this->t('Entities may be overwritten') . '

', + ]; + $form = parent::buildForm($form, $form_state); + $form['actions']['submit']['#submit'] = ['::submitConfirmIdConflictForm']; + $form['actions']['submit']['#value'] = $this->t('I acknowledge I may lose data, continue anyway.'); + + $form = $this->conflictsForm($form, $form_state, $conflicts); + return $this->i18nWarningForm($form, $form_state, $i18n); + } + + /** + * Build the markup for conflict warnings. + * + * @param array $form + * An associative array containing the structure of the form. + * @param \Drupal\Core\Form\FormStateInterface $form_state + * The current state of the form. + * @param array + * The conflicted migration labels. + * + * @return array + * The form structure. + */ + protected function conflictsForm(array &$form, FormStateInterface $form_state, $conflicts) { + if (empty($conflicts)) { + return $form; + } + + $form['warning']['#markup'] .= '

' . $this->t('Upgrades work on brand new sites, or sites with previously upgraded data. However, it looks like you have content in your site that is unknown to the migrate system; perhaps manually added. These new entities may be overwritten if you run this upgrade. For more information, see the online handbook entry for handling migration conflicts..', [':id-conflicts-handbook' => 'https://www.drupal.org/docs/8/upgrade/known-issues-when-upgrading-from-drupal-6-or-7-to-drupal-8#id_conflicts']) . '

'; + + sort($conflicts); + $form['conflicts'] = [ + '#title' => $this->t('These conflicting entities are of the following types:'), + '#theme' => 'item_list', + '#items' => $conflicts, + ]; + + return $form; + } + + /** + * Build the markup for i18n warnings. + * + * @param array $form + * An associative array containing the structure of the form. + * @param \Drupal\Core\Form\FormStateInterface $form_state + * The current state of the form. + * @param array + * The internationalization migration labels. + * + * @return array + * The form structure. + */ + protected function i18nWarningForm(array &$form, FormStateInterface $form_state, $i18n) { + if (empty($i18n)) { + return $form; + } + + $form['warning']['#markup'] .= '

' . $this->t('It looks like you are migrating translated content. Be extra cautious and make sure you aren\'t introducting any conflicts. For more on the subject, see the online handbook entry for handling migration conflicts..handling migration conflicts..', [':id-conflicts-handbook' => 'https://www.drupal.org/docs/8/upgrade/known-issues-when-upgrading-from-drupal-6-or-7-to-drupal-8#id_conflicts']) . '

'; + + sort($i18n); + $form['i18n'] = [ + '#title' => $this->t('These translatable entities are of the following types:'), + '#theme' => 'item_list', + '#items' => $i18n, + ]; + + return $form; + } + + /** + * Submission handler for the confirmation form. + * + * @param array $form + * An associative array containing the structure of the form. + * @param \Drupal\Core\Form\FormStateInterface $form_state + * The current state of the form. + */ + public function submitConfirmIdConflictForm(array &$form, FormStateInterface $form_state) { $form_state->set('step', 'confirm'); $form_state->setRebuild(); } @@ -610,4 +743,31 @@ public function getConfirmText() { return $this->t('Perform upgrade'); } + /** + * Build conflict label. + * + * @param \Drupal\migrate\Plugin\MigrationInterface $migration + * + * @TODO: This is a duplicate of MigrateIdAuditor::buildConflict() and should + * eventually be removed in https://www.drupal.org/node/2905759 + * + * @return array + */ + protected function buildConflict(MigrationInterface $migration) { + $conflict = []; + $base_id = $migration->getBaseId(); + $label = $migration->label(); + if (is_string($label)) { + $conflict[$base_id] = $label; + } + elseif ($label instanceof TranslatableMarkup) { + $conflict[$base_id] = $label->render(); + if (isset($label->getArguments()['@label'])) { + $conflict[$base_id] = $label->getArguments()['@label']; + } + } + + return $conflict; + } + } diff --git a/core/modules/migrate_drupal_ui/tests/src/Functional/MigrateUpgradeTestBase.php b/core/modules/migrate_drupal_ui/tests/src/Functional/MigrateUpgradeTestBase.php index 50d8e9d..72ed44c 100644 --- a/core/modules/migrate_drupal_ui/tests/src/Functional/MigrateUpgradeTestBase.php +++ b/core/modules/migrate_drupal_ui/tests/src/Functional/MigrateUpgradeTestBase.php @@ -147,8 +147,10 @@ public function testMigrateUpgrade() { $this->drupalPostForm(NULL, $edits, t('Review upgrade')); $this->assertResponse(200); - $this->assertText('Are you sure?'); + $this->assertSession()->pageTextContains('Entities may be overwritten'); + $this->drupalPostForm(NULL, [], t('I acknowledge I may lose data, continue anyway.')); // Ensure we get errors about missing modules. + $this->assertSession()->pageTextContains('Upgrade analysis report'); $this->assertSession()->pageTextContains(t('Source module not found for migration_provider_no_annotation.')); $this->assertSession()->pageTextContains(t('Source module not found for migration_provider_test.')); $this->assertSession()->pageTextContains(t('Destination module not found for migration_provider_test')); @@ -166,8 +168,10 @@ public function testMigrateUpgrade() { $this->drupalPostForm(NULL, $edits, t('Review upgrade')); $this->assertSession()->statusCodeEquals(200); - $this->assertSession()->pageTextContains('Are you sure?'); + $this->assertSession()->pageTextContains('Entities may be overwritten'); + $this->drupalPostForm(NULL, [], t('I acknowledge I may lose data, continue anyway.')); // Ensure there are no errors about the missing modules from the test module. + $this->assertSession()->pageTextContains('Upgrade analysis report'); $this->assertSession()->pageTextNotContains(t('Source module not found for migration_provider_no_annotation.')); $this->assertSession()->pageTextNotContains(t('Source module not found for migration_provider_test.')); $this->assertSession()->pageTextNotContains(t('Destination module not found for migration_provider_test')); diff --git a/core/modules/node/migration_templates/d6_node.yml b/core/modules/node/migration_templates/d6_node.yml index 56d0459..1bcae10 100644 --- a/core/modules/node/migration_templates/d6_node.yml +++ b/core/modules/node/migration_templates/d6_node.yml @@ -3,6 +3,7 @@ label: Nodes migration_tags: - Drupal 6 deriver: Drupal\node\Plugin\migrate\D6NodeDeriver +audit_ids: true source: plugin: d6_node process: diff --git a/core/modules/node/migration_templates/d6_node_revision.yml b/core/modules/node/migration_templates/d6_node_revision.yml index f4ff301..26c2f37 100644 --- a/core/modules/node/migration_templates/d6_node_revision.yml +++ b/core/modules/node/migration_templates/d6_node_revision.yml @@ -2,6 +2,7 @@ id: d6_node_revision label: Node revisions migration_tags: - Drupal 6 +audit_ids: true deriver: Drupal\node\Plugin\migrate\D6NodeDeriver source: plugin: d6_node_revision diff --git a/core/modules/node/migration_templates/d7_node.yml b/core/modules/node/migration_templates/d7_node.yml index 5de3055..824662f 100644 --- a/core/modules/node/migration_templates/d7_node.yml +++ b/core/modules/node/migration_templates/d7_node.yml @@ -3,6 +3,7 @@ label: Nodes migration_tags: - Drupal 7 deriver: Drupal\node\Plugin\migrate\D7NodeDeriver +audit_ids: true source: plugin: d7_node process: diff --git a/core/modules/node/migration_templates/d7_node_revision.yml b/core/modules/node/migration_templates/d7_node_revision.yml index c6081ef..020e287 100644 --- a/core/modules/node/migration_templates/d7_node_revision.yml +++ b/core/modules/node/migration_templates/d7_node_revision.yml @@ -2,6 +2,7 @@ id: d7_node_revision label: Node revisions migration_tags: - Drupal 7 +audit_ids: true deriver: Drupal\node\Plugin\migrate\D7NodeDeriver source: plugin: d7_node_revision diff --git a/core/modules/taxonomy/migration_templates/d6_taxonomy_term.yml b/core/modules/taxonomy/migration_templates/d6_taxonomy_term.yml index e3c3e3d..7ed88a7 100644 --- a/core/modules/taxonomy/migration_templates/d6_taxonomy_term.yml +++ b/core/modules/taxonomy/migration_templates/d6_taxonomy_term.yml @@ -2,6 +2,7 @@ id: d6_taxonomy_term label: Taxonomy terms migration_tags: - Drupal 6 +audit_ids: true source: plugin: d6_taxonomy_term process: diff --git a/core/modules/taxonomy/migration_templates/d6_term_node_revision.yml b/core/modules/taxonomy/migration_templates/d6_term_node_revision.yml index 91c8362..974004a 100644 --- a/core/modules/taxonomy/migration_templates/d6_term_node_revision.yml +++ b/core/modules/taxonomy/migration_templates/d6_term_node_revision.yml @@ -2,6 +2,7 @@ id: d6_term_node_revision label: Term/node relationship revisions migration_tags: - Drupal 6 +audit_ids: true deriver: Drupal\taxonomy\Plugin\migrate\D6TermNodeDeriver source: plugin: d6_term_node_revision diff --git a/core/modules/taxonomy/migration_templates/d7_taxonomy_term.yml b/core/modules/taxonomy/migration_templates/d7_taxonomy_term.yml index 99004df..e518ead 100644 --- a/core/modules/taxonomy/migration_templates/d7_taxonomy_term.yml +++ b/core/modules/taxonomy/migration_templates/d7_taxonomy_term.yml @@ -3,6 +3,7 @@ label: Taxonomy terms migration_tags: - Drupal 7 deriver: Drupal\taxonomy\Plugin\migrate\D7TaxonomyTermDeriver +audit_ids: true source: plugin: d7_taxonomy_term process: diff --git a/core/modules/user/migration_templates/d6_user.yml b/core/modules/user/migration_templates/d6_user.yml index d58607b..034ce22 100644 --- a/core/modules/user/migration_templates/d6_user.yml +++ b/core/modules/user/migration_templates/d6_user.yml @@ -2,6 +2,7 @@ id: d6_user label: User accounts migration_tags: - Drupal 6 +audit_ids: true source: plugin: d6_user process: diff --git a/core/modules/user/migration_templates/d7_user.yml b/core/modules/user/migration_templates/d7_user.yml index ae52384..9babed3 100644 --- a/core/modules/user/migration_templates/d7_user.yml +++ b/core/modules/user/migration_templates/d7_user.yml @@ -3,6 +3,7 @@ label: User accounts migration_tags: - Drupal 7 class: Drupal\user\Plugin\migrate\User +audit_ids: true source: plugin: d7_user process: diff --git a/core/modules/user/src/Plugin/migrate/destination/EntityUser.php b/core/modules/user/src/Plugin/migrate/destination/EntityUser.php index 7c2c81a..d802653 100644 --- a/core/modules/user/src/Plugin/migrate/destination/EntityUser.php +++ b/core/modules/user/src/Plugin/migrate/destination/EntityUser.php @@ -127,4 +127,18 @@ protected function processStubRow(Row $row) { } } + /** + * {@inheritdoc} + */ + public function getHighestDestinationId($field_name) { + $found = parent::getHighestDestinationId($field_name); + + // Every Drupal site must have a user with UID of 1 and it's normal for + // migrations to overwrite this user. + if ($found == 1) { + return 0; + } + return $found; + } + }