diff --git a/core/modules/migrate/src/Row.php b/core/modules/migrate/src/Row.php index 4be9712..7b3669e 100644 --- a/core/modules/migrate/src/Row.php +++ b/core/modules/migrate/src/Row.php @@ -176,9 +176,12 @@ public function setSourceProperty($property, $data) { /** * Freezes the source. + * + * @return $this */ public function freezeSource() { $this->frozen = TRUE; + return $this; } /** diff --git a/core/modules/migrate/src/Source.php b/core/modules/migrate/src/Source.php index be4368b..f55cdca 100644 --- a/core/modules/migrate/src/Source.php +++ b/core/modules/migrate/src/Source.php @@ -32,7 +32,7 @@ class Source implements \Iterator, \Countable { * * @var array */ - protected $currentIds; + protected $currentSourceIds; /** * Number of rows intentionally ignored (prepareRow() returned FALSE) @@ -51,9 +51,12 @@ class Source implements \Iterator, \Countable { /** * The high water mark at the beginning of the import operation. * - * @var + * If the source has a property for tracking changes (like Drupal ha + * node.changed) then this is the highest value of those imported so far. + * + * @var int */ - protected $originalHighWater = ''; + protected $originalHighWater; /** * List of source IDs to process. @@ -117,15 +120,26 @@ class Source implements \Iterator, \Countable { protected $idMap; /** + * The name and type of the highwater property in the source. + * * @var array + * + * @see $originalHighwater */ protected $highWaterProperty; /** - * Getter for currentIds data member. + * The result of the last prepareRow call. + * + * @var bool + */ + protected $prepareResult; + + /** + * Getter for currentSourceIds data member. */ public function getCurrentIds() { - return $this->currentIds; + return $this->currentSourceIds; } /** @@ -262,7 +276,7 @@ public function current() { * preferable. */ public function key() { - return serialize($this->currentIds); + return serialize($this->currentSourceIds); } /** @@ -297,128 +311,91 @@ public function rewind() { /** * {@inheritdoc} + * + * The migration iterates over rows returned by the source plugin. This + * method determines the next row which will be processed and imported into + * the system. + * + * The method tracks the source and destination IDs using the ID map plugin. */ public function next() { - $this->currentIds = NULL; + $this->currentSourceIds = NULL; $this->currentRow = NULL; $source_configuration = $this->migration->get('source'); - while ($this->getIterator()->valid()) { + // In order to find the next row we want to process, we ask the source + // plugin for the next possible row. + while (!isset($this->currentRow) && $this->getIterator()->valid()) { $row_data = $this->getIterator()->current() + $source_configuration; $this->getIterator()->next(); $row = new Row($row_data, $this->migration->getSourcePlugin()->getIds(), $this->migration->get('destinationIds')); + if ($this->prepareRow($row) === FALSE) { + continue; + } // Populate the source key for this row. - $this->currentIds = $row->getSourceIdValues(); + $this->currentSourceIds = $row->getSourceIdValues(); // Pick up the existing map row, if any, unless getNextRow() did it. - if (!$this->mapRowAdded && ($id_map = $this->idMap->getRowBySource($this->currentIds))) { + if (!$this->mapRowAdded && ($id_map = $this->idMap->getRowBySource($this->currentSourceIds))) { $row->setIdMap($id_map); } - // First, determine if this row should be passed to prepareRow(), or - // skipped entirely. The rules are: - // 1. If there's an explicit idlist, that's all we care about (ignore - // high waters and map rows). - $prepared = FALSE; - if (!empty($this->idList)) { - if (in_array(reset($this->currentIds), $this->idList)) { - // In the list, fall through. - } - else { - // Not in the list, skip it. - continue; - } - } - // 2. If the row is not in the map (we have never tried to import it - // before), we always want to try it. - elseif (!$row->getIdMap()) { - // Fall through - } - // 3. If the row is marked as needing update, pass it. - elseif ($row->needsUpdate()) { - // Fall through. - } - // 4. At this point, we have a row which has previously been imported and - // not marked for update. If we're not using high water marks, then we - // will not take this row. Except, if we're looking for changes in the - // data, we need to go through prepareRow() before we can decide to - // skip it. - elseif (!empty($this->highWaterProperty['field'])) { - if ($this->trackChanges) { - if ($this->prepareRow($row) !== FALSE) { - if ($row->changed()) { - // This is a keeper - $this->currentRow = $row; - break; - } - else { - // No change, skip it. - continue; - } - } - else { - // prepareRow() told us to skip it. - continue; - } - } - else { - // No high water and not tracking changes, skip. - continue; - } - } - // 5. The initial high water mark, before anything is migrated, is ''. We - // want to make sure we don't mistakenly skip rows with a high water - // field value of 0, so explicitly handle '' here. - elseif ($this->originalHighWater === '') { - // Fall through + $id_in_the_list = $this->idList && in_array(reset($this->currentSourceIds), $this->idList); + // In case we have specified an ID list, but the ID given by the source is + // not in there, we skip the row. + if ($this->idList && !$id_in_the_list) { + continue; } - // 6. So, we are using high water marks. Take the row if its high water - // field value is greater than the saved mark, otherwise skip it. - else { - // Call prepareRow() here, in case the highWaterField needs preparation. - if ($this->prepareRow($row) !== FALSE) { - if ($row->getSourceProperty($this->highWaterProperty['name']) > $this->originalHighWater) { - $this->currentRow = $row; - break; - } - else { - // Skip. - continue; - } - } - $prepared = TRUE; - } - - // Allow the Migration to prepare this row. prepareRow() can return - // boolean FALSE to ignore this row. - if (!$prepared) { - if ($this->prepareRow($row) !== FALSE) { - // Finally, we've got a keeper. - $this->currentRow = $row; - break; - } - else { - $this->currentRow = NULL; - } + // Check whether the row needs processing. + // 1. Explicitly specified IDs. + // 2. This row has not been imported yet. + // 3. Explicitly set to update. + // 4. The row is newer than the current highwater mark. + // 5. If no such property exists then try by checking the hash of the row. + if ($id_in_the_list || !$row->getIdMap() || $row->needsUpdate() || $this->aboveHighwater($row) || $this->hasRowChanged($row) ) { + $this->currentRow = $row->freezeSource(); } } - if ($this->currentRow) { - $this->currentRow->freezeSource(); - } - else { - $this->currentIds = NULL; - } } /** - * Source classes should override this as necessary and manipulate $keep. + * Check if the incoming data is newer than what we've previously imported. + * + * @param \Drupal\migrate\Row $row + * The row we're importing. + * + * @return bool + * TRUE if the highwater value in the row is greater than our current value. + */ + protected function aboveHighwater(Row $row) { + return $this->highWaterProperty && $row->getSourceProperty($this->highWaterProperty['name']) > $this->originalHighWater; + } + + /** + * Check if the row has changed. + * + * @param \Drupal\migrate\Row $row + * The row we're importing. + * + * @return bool + * TRUE if the row has changed otherwise FALSE. + */ + protected function hasRowChanged(Row $row) { + // We only use the row changed if we're not using a highwater mark. + return !$this->highWaterProperty && $this->trackChanges && $row->changed(); + } + + /** + * Add additional data to the row. * * @param \Drupal\migrate\Row $row * The row object. * * @return bool * TRUE if we're to process the row otherwise FALSE. + * + * @ee \Drupal\migrate\MigrateSourceInterface::prepareRow() */ protected function prepareRow(Row $row) { // We're explicitly skipping this row - keep track in the map table. @@ -426,21 +403,19 @@ protected function prepareRow(Row $row) { // Make sure we replace any previous messages for this item with any // new ones. $id_map = $this->migration->getIdMap(); - $id_map->delete($this->currentIds, TRUE); + $id_map->delete($this->currentSourceIds, TRUE); $this->migrateExecutable->saveQueuedMessages(); $id_map->saveIdMapping($row, array(), MigrateIdMapInterface::STATUS_IGNORED, $this->migrateExecutable->rollbackAction); $this->numIgnored++; $this->currentRow = NULL; - $this->currentIds = NULL; + $this->currentSourceIds = NULL; } - else { + elseif ($this->trackChanges) { // When tracking changed data, We want to quietly skip (rather than // "ignore") rows with changes. The caller needs to make that decision, // so we need to provide them with the necessary information (before and // after hashes). - if ($this->trackChanges) { - $row->rehash(); - } + $row->rehash(); } $this->numProcessed++; return $result; diff --git a/core/modules/migrate/tests/src/Unit/MigrateSourceTest.php b/core/modules/migrate/tests/src/Unit/MigrateSourceTest.php new file mode 100644 index 0000000..3988be2 --- /dev/null +++ b/core/modules/migrate/tests/src/Unit/MigrateSourceTest.php @@ -0,0 +1,140 @@ + 'test_migration', + 'source' => [], + ]; + + protected $row = ['test_sourceid1' => '1', 'timestamp' => 500]; + protected $sourceIds = ['test_sourceid1' => 'test_sourceid1']; + + /** + * Test that $row->needsUpdate() works as expected. + */ + public function testNextNeedsUpdate() { + $migration = $this->getMigration(); + $migrate_executable = $this->getMigrateExecutable($migration); + $source = new Source($migration, $migrate_executable); + + // $row->needsUpdate() === TRUE so we get a row. + $source->rewind(); + $this->assertTrue(is_a($source->current(), 'Drupal\migrate\Row'), '$row->needsUpdate() is TRUE so we got a row.'); + + // Test that we don't get a row when the incoming row is marked as imported. + $this->changeDefaultRowStatus(MigrateIdMapInterface::STATUS_IMPORTED); + $source->rewind(); + $this->assertNull($source->current(), 'Row was already imported, should be NULL'); + } + + /** + * Test next with highwater marks. + */ + public function testNextHighwater() { + + $migration = $this->getMigration(); + + // Update the idMap for Source so the default is that the row has already + // been imported. This allows us to use the highwater mark to decide on the + // outcome of whether we choose to import the row. + $this->changeDefaultRowStatus(MigrateIdMapInterface::STATUS_IMPORTED); + $migrate_executable = $this->getMigrateExecutable($migration); + $source = new Source($migration, $migrate_executable); + + // Set a highwater property field for source. Now we should have a row + // because the row timestamp is greater than the current highwater mark. + $this->migrationConfiguration['highWaterProperty'] = ['name' => 'timestamp']; + $source->rewind(); + $this->assertTrue(is_a($source->current(), 'Drupal\migrate\Row'), 'Incoming row timestamp is greater than current highwater mark.'); + + // Set the originalHighwater to something higher than our timestamp. + $migration + ->expects($this->any()) + ->method('getHighwater') + ->willReturn($this->row['timestamp'] + 1); + + // The current highwater mark is now higher than the row timestamp so no row + // is expected. + $source->rewind(); + $this->assertNull($source->current(), 'Original highwater mark is higher than incoming row timestamp.'); + } + + /** + * {@inheritdoc} + */ + protected function getMigration() { + $migration = parent::getMigration(); + + $iterator = new \ArrayIterator([$this->row]); + + $source_plugin = $this->getMock('Drupal\migrate\Plugin\MigrateSourceInterface'); + $source_plugin + ->expects($this->once()) + ->method('getIterator') + ->willReturn($iterator); + $source_plugin + ->expects($this->any()) + ->method('getIds') + ->willReturn($this->sourceIds); + + $migration + ->expects($this->any()) + ->method('getSourcePlugin') + ->willReturn($source_plugin); + + return $migration; + } + + /** + * Get a mock executable for the test. + * + * @param \Drupal\migrate\Entity\MigrationInterface $migration + * The migration entity. + * + * @return \Drupal\migrate\MigrateExecutable + * The migrate executable. + */ + protected function getMigrateExecutable($migration) { + $message = $this->getMock('Drupal\migrate\MigrateMessageInterface'); + return new MigrateExecutable($migration, $message); + } + + /** + * A helper to change the default imported status for new rows. + * + * @param int $status + * A status from MigrateMapIdInterface + * + * @throws \Exception + */ + protected function changeDefaultRowStatus($status) { + if (!isset($this->idMap)) { + throw new \Exception('You must call $this->getMigration() first to create a migration before changing the idMap.'); + } + $id_map_array = [ + 'original_hash' => '', + 'hash' => '', + 'source_row_status' => $status, + ]; + $this->idMap + ->expects($this->any()) + ->method('getRowBySource') + ->willReturn($id_map_array); + } +} diff --git a/core/vendor/phpunit/php-code-coverage/src/CodeCoverage.php b/core/vendor/phpunit/php-code-coverage/src/CodeCoverage.php index 06161e2..89adcfd 100644 --- a/core/vendor/phpunit/php-code-coverage/src/CodeCoverage.php +++ b/core/vendor/phpunit/php-code-coverage/src/CodeCoverage.php @@ -627,7 +627,7 @@ private function addUncoveredFilesFromWhitelist() } } } - +return; $this->append($data, 'UNCOVERED_FILES_FROM_WHITELIST'); }