only in patch2: unchanged: --- a/core/modules/migrate/migrate.services.yml +++ b/core/modules/migrate/migrate.services.yml @@ -30,3 +30,5 @@ services: plugin.manager.migration: class: Drupal\migrate\Plugin\MigrationPluginManager arguments: ['@module_handler', '@cache.discovery_migration', '@language_manager'] + migrate.id_auditor: + class: Drupal\migrate\MigrateIdAuditor only in patch2: unchanged: --- /dev/null +++ b/core/modules/migrate/src/MigrateIdAuditor.php @@ -0,0 +1,35 @@ +getDestinationPlugin(); + if ($destination instanceof MigrateIdAuditInterface) { + if ($destination->unsafeIdsExist($migration->getIdMap())) { + $ret[$destination->entityTypeId()] = TRUE; + } + } + } + return array_keys($ret); + } + +} only in patch2: unchanged: --- /dev/null +++ b/core/modules/migrate/src/Plugin/MigrateIdAuditInterface.php @@ -0,0 +1,39 @@ +getSettings(); } + /** + * Get the highest ID that exists for this destination. + * + * This is not the highest ID that has been migrated, but the highest ID + * that exists in the destination, eg: highest node ID on the site. + * + * @return int + * The highest ID value found. If no IDs at all are found, or if the + * concept of a highest ID is not meaningful, zero should be returned. + */ + protected function highestDestinationId() { + $query = $this->storage->getQuery() + ->sort($this->getHighestIdField(), 'DESC') + ->range(0, 1); + $found = $query->execute(); + return (int) reset($found); + } + + /** + * {@inheritdoc} + */ + public function getHighestIdField() { + return $this->getKey('id'); + } + + /** + * {@inheritdoc} + */ + public function unsafeIdsExist(MigrateIdMapInterface $id_map) { + // If IDs are are audited, see if there are any conflicts. + if (!empty($this->migration->getPluginDefinition()['audit_ids'])) { + if (!($id_map instanceof MigrateMaxIdInterface)) { + // We don't know how to audit IDs without a cooperating ID map. + return FALSE; + } + $highestMigrated = $id_map->getMaxId($this->getHighestIdField()); + if ($this->highestDestinationId() > $highestMigrated) { + // There's a new ID that we might conflict with! + return TRUE; + } + } + } + + /** + * {@inheritdoc} + */ + public function entityTypeId() { + return $this->storage->getEntityTypeId(); + } + } only in patch2: unchanged: --- a/core/modules/migrate/src/Plugin/migrate/destination/EntityRevision.php +++ b/core/modules/migrate/src/Plugin/migrate/destination/EntityRevision.php @@ -78,4 +78,11 @@ public function getIds() { throw new MigrateException('This entity type does not support revisions.'); } + /** + * {@inheritdoc} + */ + public function getHighestIdField() { + return $this->getKey('revision'); + } + } only in patch2: unchanged: --- a/core/modules/migrate/src/Plugin/migrate/id_map/Sql.php +++ b/core/modules/migrate/src/Plugin/migrate/id_map/Sql.php @@ -6,11 +6,14 @@ use Drupal\Core\Field\BaseFieldDefinition; use Drupal\Core\Plugin\ContainerFactoryPluginInterface; use Drupal\Core\Plugin\PluginBase; +use Drupal\migrate\Plugin\MigrateMaxIdInterface; use Drupal\migrate\Plugin\MigrationInterface; use Drupal\migrate\Event\MigrateIdMapMessageEvent; use Drupal\migrate\MigrateException; use Drupal\migrate\MigrateMessageInterface; use Drupal\migrate\Plugin\MigrateIdMapInterface; +use Drupal\migrate\Plugin\MigrationPluginManager; +use Drupal\migrate\Plugin\MigrationPluginManagerInterface; use Drupal\migrate\Row; use Drupal\migrate\Event\MigrateEvents; use Drupal\migrate\Event\MigrateMapSaveEvent; @@ -26,7 +29,7 @@ * * @PluginID("sql") */ -class Sql extends PluginBase implements MigrateIdMapInterface, ContainerFactoryPluginInterface { +class Sql extends PluginBase implements MigrateIdMapInterface, ContainerFactoryPluginInterface, MigrateMaxIdInterface { /** * Column name of hashed source id values. @@ -41,6 +44,13 @@ class Sql extends PluginBase implements MigrateIdMapInterface, ContainerFactoryP protected $eventDispatcher; /** + * The migration plugin manager to have access to the list of migrations. + * + * @var \Drupal\migrate\Plugin\MigrationPluginManagerInterface + */ + protected $migrationPluginManager; + + /** * The migration map table name. * * @var string @@ -151,11 +161,16 @@ class Sql extends PluginBase implements MigrateIdMapInterface, ContainerFactoryP * The configuration for the plugin. * @param \Drupal\migrate\Plugin\MigrationInterface $migration * The migration to do. + * @param \Symfony\Component\EventDispatcher\EventDispatcherInterface $event_dispatcher + * The event dispatcher. + * @param \Drupal\migrate\Plugin\MigrationPluginManagerInterface $migration_plugin_manager + * The migration plugin manager. */ - public function __construct(array $configuration, $plugin_id, $plugin_definition, MigrationInterface $migration, EventDispatcherInterface $event_dispatcher) { + public function __construct(array $configuration, $plugin_id, $plugin_definition, MigrationInterface $migration, EventDispatcherInterface $event_dispatcher, MigrationPluginManagerInterface $migration_plugin_manager) { parent::__construct($configuration, $plugin_id, $plugin_definition); $this->migration = $migration; $this->eventDispatcher = $event_dispatcher; + $this->migrationPluginManager = $migration_plugin_manager; } /** @@ -167,7 +182,8 @@ public static function create(ContainerInterface $container, array $configuratio $plugin_id, $plugin_definition, $migration, - $container->get('event_dispatcher') + $container->get('event_dispatcher'), + $container->get('plugin.manager.migration') ); } @@ -923,4 +939,49 @@ public function valid() { return $this->currentRow !== FALSE; } + /** + * {@inheritdoc} + */ + public function getMaxId($field) { + $sql_field = $this->destinationIdFields()[$field]; + $migration_id = $this->migration->id(); + + // List of mapping tables to look in for the highest ID. + $map_tables = [ + $migration_id => $this->mapTableName(), + ]; + + // If there's a bundle, it means we have a derived migration and we need to + // find all the mapping tables from the related derived migrations. + if ($base_id = substr($migration_id, 0, strpos($migration_id, ':'))) { + $migrations = $this->migrationPluginManager->getDefinitions(); + foreach ($migrations as $migration_id => $migration) { + if ($migration['id'] == $base_id) { + // Get this derived migration's mapping table and add it to the list + // of mapping tables to look in for the highest ID. + $migration['id'] = $migration_id; + $stub = $this->migrationPluginManager->createStubMigration($migration); + $map_tables[$migration_id] = $stub->getIdMap()->mapTableName(); + } + } + } + + // Look in all the derived migrations' mapping tables for their highest ID. + $ids = []; + foreach ($map_tables as $map_table) { + if (!$this->getDatabase()->schema()->tableExists($map_table)) { + return 0; + } + + $query = $this->getDatabase()->select($map_table, 'map') + ->fields('map', [$sql_field]) + ->orderBy($sql_field, 'DESC') + ->range(0, 1); + $ids[] = $query->execute()->fetchField(); + } + + // Return the highest of the highest IDs. + return max($ids); + } + } only in patch2: unchanged: --- a/core/modules/migrate/tests/src/Unit/MigrateSqlIdMapEnsureTablesTest.php +++ b/core/modules/migrate/tests/src/Unit/MigrateSqlIdMapEnsureTablesTest.php @@ -232,7 +232,8 @@ protected function runEnsureTablesTest($schema) { ->willReturn($plugin); /** @var \Symfony\Component\EventDispatcher\EventDispatcherInterface $event_dispatcher */ $event_dispatcher = $this->getMock('Symfony\Component\EventDispatcher\EventDispatcherInterface'); - $map = new TestSqlIdMap($database, [], 'sql', [], $migration, $event_dispatcher); + $migration_plugin_manager = $this->getMock('Drupal\migrate\Plugin\MigrationPluginManagerInterface'); + $map = new TestSqlIdMap($database, [], 'sql', [], $migration, $event_dispatcher, $migration_plugin_manager); $map->getDatabase(); } only in patch2: unchanged: --- a/core/modules/migrate/tests/src/Unit/MigrateSqlIdMapTest.php +++ b/core/modules/migrate/tests/src/Unit/MigrateSqlIdMapTest.php @@ -111,8 +111,9 @@ protected function getIdMap() { ->method('getDestinationPlugin') ->willReturn($plugin); $event_dispatcher = $this->getMock('Symfony\Component\EventDispatcher\EventDispatcherInterface'); + $migration_plugin_manager = $this->getMock('Drupal\migrate\Plugin\MigrationPluginManagerInterface'); - $id_map = new TestSqlIdMap($this->database, [], 'sql', [], $migration, $event_dispatcher); + $id_map = new TestSqlIdMap($this->database, [], 'sql', [], $migration, $event_dispatcher, $migration_plugin_manager); $migration ->method('getIdMap') ->willReturn($id_map); only in patch2: unchanged: --- a/core/modules/migrate/tests/src/Unit/TestSqlIdMap.php +++ b/core/modules/migrate/tests/src/Unit/TestSqlIdMap.php @@ -6,6 +6,7 @@ use Drupal\migrate\Plugin\MigrationInterface; use Drupal\migrate\MigrateException; use Drupal\migrate\Plugin\migrate\id_map\Sql; +use Drupal\migrate\Plugin\MigrationPluginManagerInterface; use Symfony\Component\EventDispatcher\EventDispatcherInterface; /** @@ -28,10 +29,12 @@ class TestSqlIdMap extends Sql implements \Iterator { * The migration to do. * @param \Symfony\Component\EventDispatcher\EventDispatcherInterface $event_dispatcher * The event dispatcher service. + * @param \Drupal\migrate\Plugin\MigrationPluginManagerInterface $migration_plugin_manager + * The migration plugin manager. */ - public function __construct(Connection $database, array $configuration, $plugin_id, $plugin_definition, MigrationInterface $migration, EventDispatcherInterface $event_dispatcher) { + public function __construct(Connection $database, array $configuration, $plugin_id, $plugin_definition, MigrationInterface $migration, EventDispatcherInterface $event_dispatcher, MigrationPluginManagerInterface $migration_plugin_manager) { $this->database = $database; - parent::__construct($configuration, $plugin_id, $plugin_definition, $migration, $event_dispatcher); + parent::__construct($configuration, $plugin_id, $plugin_definition, $migration, $event_dispatcher, $migration_plugin_manager); } /** only in patch2: unchanged: --- a/core/modules/migrate_drupal_ui/src/Form/MigrateUpgradeForm.php +++ b/core/modules/migrate_drupal_ui/src/Form/MigrateUpgradeForm.php @@ -3,11 +3,13 @@ namespace Drupal\migrate_drupal_ui\Form; use Drupal\Core\Datetime\DateFormatterInterface; +use Drupal\Core\Entity\EntityTypeManagerInterface; use Drupal\Core\Form\ConfirmFormBase; use Drupal\Core\Form\FormStateInterface; use Drupal\Core\Render\RendererInterface; use Drupal\Core\State\StateInterface; use Drupal\Core\Url; +use Drupal\migrate\MigrateIdAuditor; use Drupal\migrate\Plugin\MigrationPluginManagerInterface; use Drupal\migrate_drupal_ui\Batch\MigrateUpgradeImportBatch; use Drupal\migrate_drupal\MigrationConfigurationTrait; @@ -713,6 +715,20 @@ class MigrateUpgradeForm extends ConfirmFormBase { protected $pluginManager; /** + * The entity type manager. + * + * @var \Drupal\Core\Entity\EntityTypeManagerInterface + */ + protected $entityTypeManager; + + /** + * The ID conflict auditor. + * + * @var \Drupal\migrate\MigrateIdAuditor $idAuditor + */ + protected $idAuditor; + + /** * Constructs the MigrateUpgradeForm. * * @param \Drupal\Core\State\StateInterface $state @@ -723,12 +739,18 @@ class MigrateUpgradeForm extends ConfirmFormBase { * The renderer service. * @param \Drupal\migrate\Plugin\MigrationPluginManagerInterface $plugin_manager * The migration plugin manager. + * @param \Drupal\Core\Entity\EntityTypeManagerInterface $entity_type_manager + * The entity type manager. + * @param \Drupal\migrate\MigrateIdAuditor $id_auditor + * The ID conflict auditor. */ - public function __construct(StateInterface $state, DateFormatterInterface $date_formatter, RendererInterface $renderer, MigrationPluginManagerInterface $plugin_manager) { + public function __construct(StateInterface $state, DateFormatterInterface $date_formatter, RendererInterface $renderer, MigrationPluginManagerInterface $plugin_manager, EntityTypeManagerInterface $entity_type_manager, MigrateIdAuditor $id_auditor) { $this->state = $state; $this->dateFormatter = $date_formatter; $this->renderer = $renderer; $this->pluginManager = $plugin_manager; + $this->entityTypeManager = $entity_type_manager; + $this->idAuditor = $id_auditor; } /** @@ -739,7 +761,9 @@ public static function create(ContainerInterface $container) { $container->get('state'), $container->get('date.formatter'), $container->get('renderer'), - $container->get('plugin.manager.migration') + $container->get('plugin.manager.migration'), + $container->get('entity_type.manager'), + $container->get('migrate.id_auditor') ); } @@ -762,6 +786,9 @@ public function buildForm(array $form, FormStateInterface $form_state) { case 'credentials': return $this->buildCredentialForm($form, $form_state); + case 'confirm_id_conflicts': + return $this->buildIdConflictForm($form, $form_state); + case 'confirm': return $this->buildConfirmForm($form, $form_state); @@ -1082,6 +1109,64 @@ public function validateCredentialForm(array &$form, FormStateInterface $form_st */ public function submitCredentialForm(array &$form, FormStateInterface $form_state) { // Indicate the next step is confirmation. + $form_state->set('step', 'confirm_id_conflicts'); + $form_state->setRebuild(); + } + + /** + * Confirmation form for ID conflicts. + * + * @param array $form + * An associative array containing the structure of the form. + * @param \Drupal\Core\Form\FormStateInterface $form_state + * The current state of the form. + * + * @return array + * The form structure. + */ + public function buildIdConflictForm(array &$form, FormStateInterface $form_state) { + // Check if there are conflicts. If none, just skip this form! + $migration_ids = array_keys($form_state->get('migrations')); + $migrations = $this->pluginManager->createInstances($migration_ids); + $type_ids = $this->idAuditor->auditIds($migrations); + if (empty($type_ids)) { + $form_state->set('step', 'confirm'); + return $this->buildForm($form, $form_state); + } + + $form = parent::buildForm($form, $form_state); + $form['actions']['submit']['#submit'] = ['::submitConfirmIdConflictForm']; + $form['actions']['submit']['#value'] = $this->t('I acknowledge I may lose data, continue anyway.'); + + $form['warning'] = [ + '#type' => 'markup', + '#markup' => '

' . $this->t('Entities may be overwritten') . '

' . + '

' . $this->t('Upgrades work on brand new sites, or sites with previously upgraded data. However, it looks like you have content in your site that is unknown to the migrate system; perhaps manually added. These new entities may be overwritten if you run this upgrade. For more information, see the online handbook entry for handling migration conflicts..', [':id-conflicts-handbook' => 'https://www.drupal.org/docs/8/upgrade/known-issues-when-upgrading-from-drupal-6-or-7-to-drupal-8#id_conflicts']) . '

', + ]; + + $items = []; + sort($type_ids); + foreach ($type_ids as $typeId) { + $items[] = $this->entityTypeManager->getDefinition($typeId)->getLabel(); + } + $form['type_list'] = [ + '#title' => $this->t('These entities are of the following types:'), + '#theme' => 'item_list', + '#items' => $items, + ]; + + return $form; + } + + /** + * Submission handler for the confirmation form. + * + * @param array $form + * An associative array containing the structure of the form. + * @param \Drupal\Core\Form\FormStateInterface $form_state + * The current state of the form. + */ + public function submitConfirmIdConflictForm(array &$form, FormStateInterface $form_state) { $form_state->set('step', 'confirm'); $form_state->setRebuild(); } only in patch2: unchanged: --- a/core/modules/migrate_drupal_ui/tests/src/Functional/MigrateUpgradeTestBase.php +++ b/core/modules/migrate_drupal_ui/tests/src/Functional/MigrateUpgradeTestBase.php @@ -37,7 +37,6 @@ 'telephone', 'aggregator', 'book', - 'forum', 'statistics', ]; @@ -111,6 +110,29 @@ protected function tearDown() { * Executes all steps of migrations upgrade. */ public function testMigrateUpgrade() { + $driver = $this->sourceDatabase->getConnectionOptions()['driver']; + $edits = $this->getUiEditsArray(); + + // Ensure submitting the form with invalid database credentials gives us a + // nice warning. + $this->drupalPostForm(NULL, [$driver . '[database]' => 'wrong'] + $edits, t('Review upgrade')); + $this->assertText('Resolve the issue below to continue the upgrade.'); + + $this->drupalPostForm(NULL, $edits, t('Review upgrade')); + $this->assertResponse(200); + $this->assertText('Are you sure?'); + $this->drupalPostForm(NULL, [], t('Perform upgrade')); + $this->assertText(t('Congratulations, you upgraded Drupal!')); + + $this->runPostMigrationTests(); + } + + /** + * Build a re-usable form $edits array for the UI. + * + * @return array $edits + */ + protected function getUiEditsArray() { $connection_options = $this->sourceDatabase->getConnectionOptions(); $this->drupalGet('/upgrade'); $this->assertText('Upgrade a site by importing it into a clean and empty new install of Drupal 8. You will lose any existing configuration once you import your site into it. See the online documentation for Drupal site upgrades for more detailed information.'); @@ -137,26 +159,20 @@ public function testMigrateUpgrade() { if (count($drivers) !== 1) { $edit['driver'] = $driver; } - $edits = $this->translatePostValues($edit); - - // Ensure submitting the form with invalid database credentials gives us a - // nice warning. - $this->drupalPostForm(NULL, [$driver . '[database]' => 'wrong'] + $edits, t('Review upgrade')); - $this->assertText('Resolve the issue below to continue the upgrade.'); - - $this->drupalPostForm(NULL, $edits, t('Review upgrade')); - $this->assertResponse(200); - $this->assertText('Are you sure?'); - $this->drupalPostForm(NULL, [], t('Perform upgrade')); - $this->assertText(t('Congratulations, you upgraded Drupal!')); + return $this->translatePostValues($edit); + } + /** + * Runs all post migration tests. + */ + protected function runPostMigrationTests() { + $version = $this->getLegacyDrupalVersion($this->sourceDatabase); // Have to reset all the statics after migration to ensure entities are // loadable. $this->resetAll(); $expected_counts = $this->getEntityCounts(); - foreach (array_keys(\Drupal::entityTypeManager() - ->getDefinitions()) as $entity_type) { + foreach (array_keys(\Drupal::entityTypeManager()->getDefinitions()) as $entity_type) { $real_count = \Drupal::entityQuery($entity_type)->count()->execute(); $expected_count = isset($expected_counts[$entity_type]) ? $expected_counts[$entity_type] : 0; $this->assertEqual($expected_count, $real_count, "Found $real_count $entity_type entities, expected $expected_count."); @@ -182,12 +198,10 @@ public function testMigrateUpgrade() { $this->fail($message); } else { - $this->pass($message); + $this->assertTrue(TRUE, $message); } } } - \Drupal::service('module_installer')->install(['forum']); - \Drupal::service('module_installer')->install(['book']); } /** only in patch2: unchanged: --- a/core/modules/migrate_drupal_ui/tests/src/Functional/d6/MigrateUpgrade6Test.php +++ b/core/modules/migrate_drupal_ui/tests/src/Functional/d6/MigrateUpgrade6Test.php @@ -17,6 +17,19 @@ class MigrateUpgrade6Test extends MigrateUpgradeTestBase { /** * {@inheritdoc} */ + public static $modules = [ + 'language', + 'content_translation', + 'migrate_drupal_ui', + 'telephone', + 'aggregator', + 'book', + 'statistics', + ]; + + /** + * {@inheritdoc} + */ protected function setUp() { parent::setUp(); $this->loadFixture(drupal_get_path('module', 'migrate_drupal') . '/tests/fixtures/drupal6.php'); @@ -40,7 +53,7 @@ protected function getEntityCounts() { 'block_content' => 2, 'block_content_type' => 1, 'comment' => 3, - 'comment_type' => 3, + 'comment_type' => 2, 'contact_form' => 5, 'configurable_language' => 5, 'editor' => 2, only in patch2: unchanged: --- /dev/null +++ b/core/modules/migrate_drupal_ui/tests/src/Functional/d6/MigrateUpgradeConflicts6Test.php @@ -0,0 +1,60 @@ +sourceDatabase->getConnectionOptions()['driver']; + $edits = $this->getUiEditsArray(); + + // Ensure submitting the form with invalid database credentials gives us a + // nice warning. + $this->drupalPostForm(NULL, [$driver . '[database]' => 'wrong'] + $edits, t('Review upgrade')); + $this->assertText('Resolve the issue below to continue the upgrade.'); + + $this->drupalPostForm(NULL, $edits, t('Review upgrade')); + $this->assertResponse(200); + $this->assertText('Are you sure?'); + $this->drupalPostForm(NULL, [], t('I acknowledge I may lose data, continue anyway.')); + $this->drupalPostForm(NULL, [], t('Perform upgrade')); + $this->assertText(t('Congratulations, you upgraded Drupal!')); + + $this->runPostMigrationTests(); + } + + /** + * {@inheritdoc} + */ + protected function getEntityCounts() { + $counts = parent::getEntityCounts(); + $counts['comment_type'] = 3; + return $counts; + } + +} only in patch2: unchanged: --- a/core/modules/migrate_drupal_ui/tests/src/Functional/d7/MigrateUpgrade7Test.php +++ b/core/modules/migrate_drupal_ui/tests/src/Functional/d7/MigrateUpgrade7Test.php @@ -45,7 +45,7 @@ protected function getEntityCounts() { 'block_content' => 1, 'block_content_type' => 1, 'comment' => 1, - 'comment_type' => 8, + 'comment_type' => 7, // Module 'language' comes with 'en', 'und', 'zxx'. Migration adds 'is'. 'configurable_language' => 4, 'contact_form' => 3, only in patch2: unchanged: --- /dev/null +++ b/core/modules/migrate_drupal_ui/tests/src/Functional/d7/MigrateUpgradeConflicts7Test.php @@ -0,0 +1,62 @@ +sourceDatabase->getConnectionOptions()['driver']; + $edits = $this->getUiEditsArray(); + + // Ensure submitting the form with invalid database credentials gives us a + // nice warning. + $this->drupalPostForm(NULL, [$driver . '[database]' => 'wrong'] + $edits, t('Review upgrade')); + $this->assertText('Resolve the issue below to continue the upgrade.'); + + $this->drupalPostForm(NULL, $edits, t('Review upgrade')); + $this->assertResponse(200); + $this->assertText('Are you sure?'); + $this->drupalPostForm(NULL, [], t('I acknowledge I may lose data, continue anyway.')); + $this->drupalPostForm(NULL, [], t('Perform upgrade')); + $this->assertText(t('Congratulations, you upgraded Drupal!')); + + $this->runPostMigrationTests(); + } + + /** + * {@inheritdoc} + */ + protected function getEntityCounts() { + $counts = parent::getEntityCounts(); + // Forum has a comment. + $counts['comment_type'] = 8; + return $counts; + } + +} only in patch2: unchanged: --- a/core/modules/user/src/Plugin/migrate/destination/EntityUser.php +++ b/core/modules/user/src/Plugin/migrate/destination/EntityUser.php @@ -127,4 +127,18 @@ protected function processStubRow(Row $row) { } } + /** + * {@inheritdoc} + */ + protected function highestDestinationId() { + $found = parent::highestDestinationId(); + + // Every Drupal site must have at least a single non-anonymous user, and + // it's normal for upgrade migrations to overwrite this user. + if ($found == 1) { + return 0; + } + return $found; + } + }