only in patch2: unchanged: --- a/core/modules/migrate/migrate.services.yml +++ b/core/modules/migrate/migrate.services.yml @@ -30,3 +30,5 @@ services: plugin.manager.migration: class: Drupal\migrate\Plugin\MigrationPluginManager arguments: ['@module_handler', '@cache.discovery_migration', '@language_manager'] + migrate.id_auditor: + class: Drupal\migrate\MigrateIdAuditor only in patch2: unchanged: --- /dev/null +++ b/core/modules/migrate/src/MigrateIdAuditor.php @@ -0,0 +1,35 @@ +getDestinationPlugin(); + if ($destination instanceof MigrateIdAuditInterface) { + if ($destination->unsafeIdsExist($migration->getIdMap())) { + $ret[$destination->entityTypeId()] = TRUE; + } + } + } + return array_keys($ret); + } + +} only in patch2: unchanged: --- /dev/null +++ b/core/modules/migrate/src/Plugin/MigrateIdAuditInterface.php @@ -0,0 +1,39 @@ +getSettings(); } + /** + * Get the highest ID that exists for this destination. + * + * This is not the highest ID that has been migrated, but the highest ID + * that exists in the destination, eg: highest node ID on the site. + * + * @return int + * The highest ID value found. If no IDs at all are found, or if the + * concept of a highest ID is not meaningful, zero should be returned. + */ + protected function highestDestinationId() { + $query = $this->storage->getQuery() + ->sort($this->getHighestIdField(), 'DESC') + ->range(0, 1); + $found = $query->execute(); + return (int) reset($found); + } + + /** + * {@inheritdoc} + */ + public function getHighestIdField() { + return $this->getKey('id'); + } + + /** + * {@inheritdoc} + */ + public function unsafeIdsExist(MigrateIdMapInterface $id_map) { + // If IDs are are audited, see if there are any conflicts. + if (!empty($this->migration->getPluginDefinition()['audit_ids'])) { + if (!($id_map instanceof MigrateMaxIdInterface)) { + // We don't know how to audit IDs without a cooperating ID map. + return FALSE; + } + $highestMigrated = $id_map->getMaxId($this->getHighestIdField()); + if ($this->highestDestinationId() > $highestMigrated) { + // There's a new ID that we might conflict with! + return TRUE; + } + } + } + + /** + * {@inheritdoc} + */ + public function entityTypeId() { + return $this->storage->getEntityTypeId(); + } + } only in patch2: unchanged: --- a/core/modules/migrate/src/Plugin/migrate/destination/EntityRevision.php +++ b/core/modules/migrate/src/Plugin/migrate/destination/EntityRevision.php @@ -78,4 +78,11 @@ public function getIds() { throw new MigrateException('This entity type does not support revisions.'); } + /** + * {@inheritdoc} + */ + public function getHighestIdField() { + return $this->getKey('revision'); + } + } only in patch2: unchanged: --- a/core/modules/migrate/src/Plugin/migrate/id_map/Sql.php +++ b/core/modules/migrate/src/Plugin/migrate/id_map/Sql.php @@ -6,6 +6,7 @@ use Drupal\Core\Field\BaseFieldDefinition; use Drupal\Core\Plugin\ContainerFactoryPluginInterface; use Drupal\Core\Plugin\PluginBase; +use Drupal\migrate\Plugin\MigrateMaxIdInterface; use Drupal\migrate\Plugin\MigrationInterface; use Drupal\migrate\Event\MigrateIdMapMessageEvent; use Drupal\migrate\MigrateException; @@ -26,7 +27,7 @@ * * @PluginID("sql") */ -class Sql extends PluginBase implements MigrateIdMapInterface, ContainerFactoryPluginInterface { +class Sql extends PluginBase implements MigrateIdMapInterface, ContainerFactoryPluginInterface, MigrateMaxIdInterface { /** * Column name of hashed source id values. @@ -923,4 +924,22 @@ public function valid() { return $this->currentRow !== FALSE; } + /** + * {@inheritdoc} + */ + public function getMaxId($field) { + if (!$this->getDatabase()->schema()->tableExists($this->mapTableName())) { + return 0; + } + + $sqlField = $this->destinationIdFields()[$field]; + + $query = $this->getDatabase()->select($this->mapTableName(), 'map') + ->fields('map', [$sqlField]) + ->orderBy($sqlField, 'DESC') + ->range(0, 1); + $found = $query->execute()->fetchField(); + return (int) $found; + } + } only in patch2: unchanged: --- a/core/modules/migrate_drupal_ui/src/Form/MigrateUpgradeForm.php +++ b/core/modules/migrate_drupal_ui/src/Form/MigrateUpgradeForm.php @@ -3,11 +3,13 @@ namespace Drupal\migrate_drupal_ui\Form; use Drupal\Core\Datetime\DateFormatterInterface; +use Drupal\Core\Entity\EntityTypeManagerInterface; use Drupal\Core\Form\ConfirmFormBase; use Drupal\Core\Form\FormStateInterface; use Drupal\Core\Render\RendererInterface; use Drupal\Core\State\StateInterface; use Drupal\Core\Url; +use Drupal\migrate\MigrateIdAuditor; use Drupal\migrate\Plugin\MigrationPluginManagerInterface; use Drupal\migrate_drupal_ui\Batch\MigrateUpgradeImportBatch; use Drupal\migrate_drupal\MigrationConfigurationTrait; @@ -713,6 +715,20 @@ class MigrateUpgradeForm extends ConfirmFormBase { protected $pluginManager; /** + * The entity type manager. + * + * @var \Drupal\Core\Entity\EntityTypeManagerInterface + */ + protected $entityTypeManager; + + /** + * The ID conflict auditor. + * + * @var \Drupal\migrate\MigrateIdAuditor $idAuditor + */ + protected $idAuditor; + + /** * Constructs the MigrateUpgradeForm. * * @param \Drupal\Core\State\StateInterface $state @@ -723,12 +739,18 @@ class MigrateUpgradeForm extends ConfirmFormBase { * The renderer service. * @param \Drupal\migrate\Plugin\MigrationPluginManagerInterface $plugin_manager * The migration plugin manager. + * @param \Drupal\Core\Entity\EntityTypeManagerInterface $entity_type_manager + * The entity type manager. + * @param \Drupal\migrate\MigrateIdAuditor $id_auditor + * The ID conflict auditor. */ - public function __construct(StateInterface $state, DateFormatterInterface $date_formatter, RendererInterface $renderer, MigrationPluginManagerInterface $plugin_manager) { + public function __construct(StateInterface $state, DateFormatterInterface $date_formatter, RendererInterface $renderer, MigrationPluginManagerInterface $plugin_manager, EntityTypeManagerInterface $entity_type_manager, MigrateIdAuditor $id_auditor) { $this->state = $state; $this->dateFormatter = $date_formatter; $this->renderer = $renderer; $this->pluginManager = $plugin_manager; + $this->entityTypeManager = $entity_type_manager; + $this->idAuditor = $id_auditor; } /** @@ -739,7 +761,9 @@ public static function create(ContainerInterface $container) { $container->get('state'), $container->get('date.formatter'), $container->get('renderer'), - $container->get('plugin.manager.migration') + $container->get('plugin.manager.migration'), + $container->get('entity_type.manager'), + $container->get('migrate.id_auditor') ); } @@ -762,6 +786,9 @@ public function buildForm(array $form, FormStateInterface $form_state) { case 'credentials': return $this->buildCredentialForm($form, $form_state); + case 'confirm_id_conflicts': + return $this->buildIdConflictForm($form, $form_state); + case 'confirm': return $this->buildConfirmForm($form, $form_state); @@ -1082,6 +1109,64 @@ public function validateCredentialForm(array &$form, FormStateInterface $form_st */ public function submitCredentialForm(array &$form, FormStateInterface $form_state) { // Indicate the next step is confirmation. + $form_state->set('step', 'confirm_id_conflicts'); + $form_state->setRebuild(); + } + + /** + * Confirmation form for ID conflicts. + * + * @param array $form + * An associative array containing the structure of the form. + * @param \Drupal\Core\Form\FormStateInterface $form_state + * The current state of the form. + * + * @return array + * The form structure. + */ + public function buildIdConflictForm(array &$form, FormStateInterface $form_state) { + // Check if there are conflicts. If none, just skip this form! + $migration_ids = array_keys($form_state->get('migrations')); + $migrations = $this->pluginManager->createInstances($migration_ids); + $type_ids = $this->idAuditor->auditIds($migrations); + if (empty($type_ids)) { + $form_state->set('step', 'confirm'); + return $this->buildForm($form, $form_state); + } + + $form = parent::buildForm($form, $form_state); + $form['actions']['submit']['#submit'] = ['::submitConfirmIdConflictForm']; + $form['actions']['submit']['#value'] = $this->t('I acknowledge I may lose data, continue anyway.'); + + $form['warning'] = [ + '#type' => 'markup', + '#markup' => '

' . $this->t('Entities may be overwritten') . '

' . + '

' . $this->t('Upgrades work on brand new sites, or sites with previously upgraded data. However, it looks like you have content in your site that is unknown to the migrate system; perhaps manually added. These new entities may be overwritten if you run this upgrade. For more information, see the online handbook entry for handling migration conflicts..', [':id-conflicts-handbook' => 'https://www.drupal.org/docs/8/upgrade/known-issues-when-upgrading-from-drupal-6-or-7-to-drupal-8#id_conflicts']) . '

', + ]; + + $items = []; + sort($type_ids); + foreach ($type_ids as $typeId) { + $items[] = $this->entityTypeManager->getDefinition($typeId)->getLabel(); + } + $form['type_list'] = [ + '#title' => $this->t('These entities are of the following types:'), + '#theme' => 'item_list', + '#items' => $items, + ]; + + return $form; + } + + /** + * Submission handler for the confirmation form. + * + * @param array $form + * An associative array containing the structure of the form. + * @param \Drupal\Core\Form\FormStateInterface $form_state + * The current state of the form. + */ + public function submitConfirmIdConflictForm(array &$form, FormStateInterface $form_state) { $form_state->set('step', 'confirm'); $form_state->setRebuild(); } only in patch2: unchanged: --- a/core/modules/migrate_drupal_ui/tests/src/Functional/MigrateUpgradeTestBase.php +++ b/core/modules/migrate_drupal_ui/tests/src/Functional/MigrateUpgradeTestBase.php @@ -37,7 +37,6 @@ 'telephone', 'aggregator', 'book', - 'forum', 'statistics', ]; @@ -111,6 +110,29 @@ protected function tearDown() { * Executes all steps of migrations upgrade. */ public function testMigrateUpgrade() { + $driver = $this->sourceDatabase->getConnectionOptions()['driver']; + $edits = $this->getUiEditsArray(); + + // Ensure submitting the form with invalid database credentials gives us a + // nice warning. + $this->drupalPostForm(NULL, [$driver . '[database]' => 'wrong'] + $edits, t('Review upgrade')); + $this->assertText('Resolve the issue below to continue the upgrade.'); + + $this->drupalPostForm(NULL, $edits, t('Review upgrade')); + $this->assertResponse(200); + $this->assertText('Are you sure?'); + $this->drupalPostForm(NULL, [], t('Perform upgrade')); + $this->assertText(t('Congratulations, you upgraded Drupal!')); + + $this->runPostMigrationTests(); + } + + /** + * Build a re-usable form $edits array for the UI. + * + * @return array $edits + */ + protected function getUiEditsArray() { $connection_options = $this->sourceDatabase->getConnectionOptions(); $this->drupalGet('/upgrade'); $this->assertText('Upgrade a site by importing it into a clean and empty new install of Drupal 8. You will lose any existing configuration once you import your site into it. See the online documentation for Drupal site upgrades for more detailed information.'); @@ -137,29 +159,23 @@ public function testMigrateUpgrade() { if (count($drivers) !== 1) { $edit['driver'] = $driver; } - $edits = $this->translatePostValues($edit); - - // Ensure submitting the form with invalid database credentials gives us a - // nice warning. - $this->drupalPostForm(NULL, [$driver . '[database]' => 'wrong'] + $edits, t('Review upgrade')); - $this->assertText('Resolve the issue below to continue the upgrade.'); - - $this->drupalPostForm(NULL, $edits, t('Review upgrade')); - $this->assertResponse(200); - $this->assertText('Are you sure?'); - $this->drupalPostForm(NULL, [], t('Perform upgrade')); - $this->assertText(t('Congratulations, you upgraded Drupal!')); + return $this->translatePostValues($edit); + } + /** + * Runs all post migration tests. + */ + protected function runPostMigrationTests() { + $version = $this->getLegacyDrupalVersion($this->sourceDatabase); // Have to reset all the statics after migration to ensure entities are // loadable. $this->resetAll(); $expected_counts = $this->getEntityCounts(); - foreach (array_keys(\Drupal::entityTypeManager() - ->getDefinitions()) as $entity_type) { + foreach (array_keys(\Drupal::entityTypeManager()->getDefinitions()) as $entity_type) { $real_count = \Drupal::entityQuery($entity_type)->count()->execute(); $expected_count = isset($expected_counts[$entity_type]) ? $expected_counts[$entity_type] : 0; - $this->assertEqual($expected_count, $real_count, "Found $real_count $entity_type entities, expected $expected_count."); + $this->assertSame($expected_count, $real_count, "Found $real_count $entity_type entities, expected $expected_count."); } $plugin_manager = \Drupal::service('plugin.manager.migration'); @@ -182,12 +198,10 @@ public function testMigrateUpgrade() { $this->fail($message); } else { - $this->pass($message); + $this->assertTrue(TRUE, $message); } } } - \Drupal::service('module_installer')->install(['forum']); - \Drupal::service('module_installer')->install(['book']); } /** only in patch2: unchanged: --- a/core/modules/migrate_drupal_ui/tests/src/Functional/d6/MigrateUpgrade6Test.php +++ b/core/modules/migrate_drupal_ui/tests/src/Functional/d6/MigrateUpgrade6Test.php @@ -17,6 +17,19 @@ class MigrateUpgrade6Test extends MigrateUpgradeTestBase { /** * {@inheritdoc} */ + public static $modules = [ + 'language', + 'content_translation', + 'migrate_drupal_ui', + 'telephone', + 'aggregator', + 'book', + 'statistics', + ]; + + /** + * {@inheritdoc} + */ protected function setUp() { parent::setUp(); $this->loadFixture(drupal_get_path('module', 'migrate_drupal') . '/tests/fixtures/drupal6.php'); @@ -40,7 +53,7 @@ protected function getEntityCounts() { 'block_content' => 2, 'block_content_type' => 1, 'comment' => 3, - 'comment_type' => 3, + 'comment_type' => 2, 'contact_form' => 5, 'configurable_language' => 5, 'editor' => 2, only in patch2: unchanged: --- /dev/null +++ b/core/modules/migrate_drupal_ui/tests/src/Functional/d6/MigrateUpgradeConflicts6Test.php @@ -0,0 +1,60 @@ +sourceDatabase->getConnectionOptions()['driver']; + $edits = $this->getUiEditsArray(); + + // Ensure submitting the form with invalid database credentials gives us a + // nice warning. + $this->drupalPostForm(NULL, [$driver . '[database]' => 'wrong'] + $edits, t('Review upgrade')); + $this->assertText('Resolve the issue below to continue the upgrade.'); + + $this->drupalPostForm(NULL, $edits, t('Review upgrade')); + $this->assertResponse(200); + $this->assertText('Are you sure?'); + $this->drupalPostForm(NULL, [], t('I acknowledge I may lose data, continue anyway.')); + $this->drupalPostForm(NULL, [], t('Perform upgrade')); + $this->assertText(t('Congratulations, you upgraded Drupal!')); + + $this->runPostMigrationTests(); + } + + /** + * {@inheritdoc} + */ + protected function getEntityCounts() { + $counts = parent::getEntityCounts(); + $counts['comment_type'] = 3; + return $counts; + } + +} only in patch2: unchanged: --- a/core/modules/migrate_drupal_ui/tests/src/Functional/d7/MigrateUpgrade7Test.php +++ b/core/modules/migrate_drupal_ui/tests/src/Functional/d7/MigrateUpgrade7Test.php @@ -45,7 +45,7 @@ protected function getEntityCounts() { 'block_content' => 1, 'block_content_type' => 1, 'comment' => 1, - 'comment_type' => 8, + 'comment_type' => 7, // Module 'language' comes with 'en', 'und', 'zxx'. Migration adds 'is'. 'configurable_language' => 4, 'contact_form' => 3, only in patch2: unchanged: --- /dev/null +++ b/core/modules/migrate_drupal_ui/tests/src/Functional/d7/MigrateUpgradeConflicts7Test.php @@ -0,0 +1,62 @@ +sourceDatabase->getConnectionOptions()['driver']; + $edits = $this->getUiEditsArray(); + + // Ensure submitting the form with invalid database credentials gives us a + // nice warning. + $this->drupalPostForm(NULL, [$driver . '[database]' => 'wrong'] + $edits, t('Review upgrade')); + $this->assertText('Resolve the issue below to continue the upgrade.'); + + $this->drupalPostForm(NULL, $edits, t('Review upgrade')); + $this->assertResponse(200); + $this->assertText('Are you sure?'); + $this->drupalPostForm(NULL, [], t('I acknowledge I may lose data, continue anyway.')); + $this->drupalPostForm(NULL, [], t('Perform upgrade')); + $this->assertText(t('Congratulations, you upgraded Drupal!')); + + $this->runPostMigrationTests(); + } + + /** + * {@inheritdoc} + */ + protected function getEntityCounts() { + $counts = parent::getEntityCounts(); + // Forum has a comment. + $counts['comment_type'] = 8; + return $counts; + } + +} only in patch2: unchanged: --- a/core/modules/shortcut/migration_templates/d7_shortcut.yml +++ b/core/modules/shortcut/migration_templates/d7_shortcut.yml @@ -2,6 +2,7 @@ id: d7_shortcut label: Shortcut links migration_tags: - Drupal 7 +audit_ids: false source: plugin: d7_shortcut constants: only in patch2: unchanged: --- a/core/modules/user/src/Plugin/migrate/destination/EntityUser.php +++ b/core/modules/user/src/Plugin/migrate/destination/EntityUser.php @@ -127,4 +127,18 @@ protected function processStubRow(Row $row) { } } + /** + * {@inheritdoc} + */ + protected function highestDestinationId() { + $found = parent::highestDestinationId(); + + // Every Drupal site must have at least a single non-anonymous user, and + // it's normal for upgrade migrations to overwrite this user. + if ($found == 1) { + return 0; + } + return $found; + } + }