diff -u b/core/modules/migrate/src/Plugin/Migration.php b/core/modules/migrate/src/Plugin/Migration.php --- b/core/modules/migrate/src/Plugin/Migration.php +++ b/core/modules/migrate/src/Plugin/Migration.php @@ -126,15 +126,6 @@ protected $destinationIds = []; /** - * Information on the property used as the high watermark. - * - * Array of 'name' & (optional) db 'alias' properties used for high watermark. - * - * @var array - */ - protected $high_water_property = []; - - /** * Indicate whether the primary system of record for this migration is the * source, or the destination (Drupal). In the source case, migration of * an existing object will completely replace the Drupal object with data from @@ -155,11 +146,6 @@ protected $sourceRowStatus = MigrateIdMapInterface::STATUS_IMPORTED; /** - * @var \Drupal\Core\KeyValueStore\KeyValueStoreInterface - */ - protected $highWaterStorage; - - /** * Track time of last import if TRUE. * * @var bool @@ -444,30 +430,17 @@ } /** - * Get the high water storage object. - * - * @return \Drupal\Core\KeyValueStore\KeyValueStoreInterface - * The storage object. - */ - protected function getHighWaterStorage() { - if (!isset($this->highWaterStorage)) { - $this->highWaterStorage = \Drupal::keyValue('migrate:high_water'); - } - return $this->highWaterStorage; - } - - /** * {@inheritdoc} */ public function getHighWater() { - return $this->getHighWaterStorage()->get($this->id()); + return $this->getSourcePlugin()->getHighWater(); } /** * {@inheritdoc} */ public function saveHighWater($high_water) { - $this->getHighWaterStorage()->set($this->id(), $high_water); + $this->getSourcePlugin()->saveHighWater($high_water); } /** @@ -703,22 +676,7 @@ * {@inheritdoc} */ public function getHighWaterProperty() { - return isset($this->getSourceConfiguration()['high_water_property']) ? $this->getSourceConfiguration()['high_water_property'] : NULL; - } - - /** - * {@inheritdoc} - */ - public function getHighWaterField() { - if ($this->getHighWaterProperty()) { - $high_water_property = $this->getHighWaterProperty(); - if (!empty($high_water_property['name'])) { - return !empty($high_water_property['alias']) ? - $high_water_property['alias'] . '.' . $high_water_property['name'] : - $high_water_property['name']; - } - } - return NULL; + return $this->getSourcePlugin()->getHighWaterProperty(); } /** diff -u b/core/modules/migrate/src/Plugin/MigrationInterface.php b/core/modules/migrate/src/Plugin/MigrationInterface.php --- b/core/modules/migrate/src/Plugin/MigrationInterface.php +++ b/core/modules/migrate/src/Plugin/MigrationInterface.php @@ -159,9 +159,11 @@ * run. If the mark is set, only content with a higher timestamp will be * imported. * - * @return int|NULL - * A Unix timestamp representing the high water mark, or NULL if no high - * water mark has been stored. + * @return int + * A Unix timestamp representing the high water mark. + * + * @deprecated in Drupal 8.2.0, will be removed in Drupal 8.3.x. Use + * Drupal\migrate\Plugin\MigrateSourceInterface::getHighWater() instead. */ public function getHighWater(); @@ -170,6 +172,9 @@ * * @param int $high_water * The high water timestamp. + * + * @deprecated in Drupal 8.2.0, will be removed in Drupal 8.3.x. Use + * Drupal\migrate\Plugin\MigrateSourceInterface::saveHighWater() instead. */ public function saveHighWater($high_water); @@ -353,20 +358,11 @@ * * @return array * The property used as the high watermark. - */ - public function getHighWaterProperty(); - - /** - * Get the name of the field used as the high watermark. - * - * The name of the field qualified with an alias if available. * - * @see Drupal\migrate\Plugin\migrate\source\SqlBase::initializeIterator() - * - * @return string|NULL - * The name of the field for the high water mark, or NULL if not set. + * @deprecated in Drupal 8.2.0, will be removed in Drupal 8.3.x. Use + * Drupal\migrate\Plugin\MigrateSourceInterface::getHighWaterProperty() instead. */ - public function getHighWaterField(); + public function getHighWaterProperty(); /** * If true, track time of last import. diff -u b/core/modules/migrate/src/Plugin/migrate/source/SqlBase.php b/core/modules/migrate/src/Plugin/migrate/source/SqlBase.php --- b/core/modules/migrate/src/Plugin/migrate/source/SqlBase.php +++ b/core/modules/migrate/src/Plugin/migrate/source/SqlBase.php @@ -212,18 +212,15 @@ } // 2. If we are using high water marks, also include rows above the mark. // But, include all rows if the high water mark is not set. - if (isset($this->configuration['high_water_property'])) { - $high_water_property = $this->configuration['high_water_property']; - if (isset($high_water_property['name']) && ($high_water = $this->migration->getHighWater()) !== '') { - $high_water_field = $this->migration->getHighWaterField(); - $conditions->condition($high_water_field, $high_water, '>'); - $this->query->orderBy($high_water_field); - $condition_added = TRUE; - } + if ($this->getHighWaterProperty() && ($high_water = $this->getHighWater()) !== '') { + $high_water_field = $this->getHighWaterField(); + $conditions->condition($high_water_field, $high_water, '>'); + $this->query->orderBy($high_water_field); } if ($condition_added) { $this->query->condition($conditions); } + return new \IteratorIterator($this->query->execute()); } diff -u b/core/modules/migrate/tests/src/Kernel/HighWaterTest.php b/core/modules/migrate/tests/src/Kernel/HighWaterTest.php --- b/core/modules/migrate/tests/src/Kernel/HighWaterTest.php +++ b/core/modules/migrate/tests/src/Kernel/HighWaterTest.php @@ -90,9 +90,9 @@ $this->assertNodeExists('Item 1'); $this->assertNodeExists('Item 2'); $this->assertNodeExists('Item 3'); - // Highwater mark is now equal to 3. - $this->assertSame('3', $this->migration->getHighWater()); + + $this->assertSame('3', $this->getMigration('high_water_test')->getSourcePlugin()->getHighWater()); // Update Item 1 setting its high_water_property to value that is below // current high water mark. diff -u b/core/modules/migrate/tests/src/Unit/MigrateSourceTest.php b/core/modules/migrate/tests/src/Unit/MigrateSourceTest.php --- b/core/modules/migrate/tests/src/Unit/MigrateSourceTest.php +++ b/core/modules/migrate/tests/src/Unit/MigrateSourceTest.php @@ -90,7 +90,7 @@ ->willReturn($id_map_array); $constructor_args = [$configuration, 'd6_action', [], $this->migration]; - $methods = ['getModuleHandler', 'fields', 'getIds', '__toString', 'getIterator', 'prepareRow', 'initializeIterator', 'calculateDependencies']; + $methods = ['getModuleHandler', 'fields', 'getIds', '__toString', 'getIterator', 'prepareRow', 'initializeIterator', 'calculateDependencies', 'getHighWater']; $source_plugin = $this->getMock('\Drupal\migrate\Plugin\migrate\source\SourcePluginBase', $methods, $constructor_args); $source_plugin @@ -138,9 +138,8 @@ * @expectedException \Drupal\migrate\MigrateException */ public function testHighwaterTrackChangesIncompatible() { - $source_config = ['track_changes' => TRUE]; - $migration_config = ['high_water_property' => ['name' => 'something']]; - $this->getSource($source_config, $migration_config); + $source_config = ['track_changes' => TRUE, 'high_water_property' => ['name' => 'something']]; + $this->getSource($source_config); } /** @@ -240,10 +239,9 @@ * @throws \Exception */ public function testNewHighwater() { - // Set a highwater property field for source. Now we should have a row // because the row timestamp is greater than the current highwater mark. - $source = $this->getSource([], ['high_water_property' => ['name' => 'timestamp']], MigrateIdMapInterface::STATUS_IMPORTED); + $source = $this->getSource(['highWaterProperty' => ['name' => 'timestamp']], [], MigrateIdMapInterface::STATUS_IMPORTED); $source->rewind(); $this->assertTrue(is_a($source->current(), 'Drupal\migrate\Row'), 'Incoming row timestamp is greater than current highwater mark so we have a row.'); diff -u b/core/modules/migrate/tests/src/Unit/MigrateSqlSourceTestCase.php b/core/modules/migrate/tests/src/Unit/MigrateSqlSourceTestCase.php --- b/core/modules/migrate/tests/src/Unit/MigrateSqlSourceTestCase.php +++ b/core/modules/migrate/tests/src/Unit/MigrateSqlSourceTestCase.php @@ -70,13 +70,6 @@ protected $plugin; /** - * The mock migration plugin. - * - * @var \Drupal\migrate\Plugin\Migration|\PHPUnit_Framework_MockObject_MockObject - */ - protected $migration; - - /** * {@inheritdoc} */ protected function setUp() { @@ -84,11 +77,14 @@ $state = $this->getMock('Drupal\Core\State\StateInterface'); $entity_manager = $this->getMock('Drupal\Core\Entity\EntityManagerInterface'); - $this->migration = $this->getMigration(); + $migration = $this->getMigration(); + $migration->expects($this->any()) + ->method('getHighWater') + ->will($this->returnValue(static::ORIGINAL_HIGH_WATER)); // Setup the plugin. $plugin_class = static::PLUGIN_CLASS; - $plugin = new $plugin_class($this->migrationConfiguration['source'], $this->migrationConfiguration['source']['plugin'], array(), $this->migration, $state, $entity_manager); + $plugin = new $plugin_class($this->migrationConfiguration['source'], $this->migrationConfiguration['source']['plugin'], array(), $migration, $state, $entity_manager); // Do some reflection to set the database and moduleHandler. $plugin_reflection = new \ReflectionClass($plugin); @@ -102,7 +98,7 @@ $module_handler_property->setValue($plugin, $module_handler); $plugin->setStringTranslation($this->getStringTranslationStub()); - $this->migration->expects($this->any()) + $migration->expects($this->any()) ->method('getSourcePlugin') ->will($this->returnValue($plugin)); $this->source = $plugin; @@ -113,18 +109,6 @@ * Tests that the source returns the same rows as expected. */ public function testRetrieval() { - - // If the high-water value is set, expect that $migration->getHighWater() - // and $migration->getHighWaterField. - if (static::ORIGINAL_HIGH_WATER !== NULL) { - $this->migration->expects($this->atLeastOnce()) - ->method('getHighWater') - ->willReturn(static::ORIGINAL_HIGH_WATER); - - $this->migration->expects($this->atLeastOnce()) - ->method('getHighWaterField') - ->will($this->returnValue('timestamp')); - } $this->assertInstanceOf(SelectInterface::class, $this->source->query()); $this->queryResultTest($this->source, $this->expectedResults); } only in patch2: unchanged: --- a/core/modules/migrate/src/MigrateExecutable.php +++ b/core/modules/migrate/src/MigrateExecutable.php @@ -256,9 +256,6 @@ public function import() { $this->handleException($e); } } - if ($high_water_property = $this->migration->getHighWaterProperty()) { - $this->migration->saveHighWater($row->getSourceProperty($high_water_property['name'])); - } // Reset row properties. unset($sourceValues, $destinationValues); @@ -350,7 +347,7 @@ public function rollback() { } // If rollback completed successfully, reset the high water mark. if ($return == MigrationInterface::RESULT_COMPLETED) { - $this->migration->saveHighWater(NULL); + $this->getSource()->saveHighWater(NULL); } // Notify modules that rollback attempt was complete. only in patch2: unchanged: --- a/core/modules/migrate/src/Plugin/MigrateSourceInterface.php +++ b/core/modules/migrate/src/Plugin/MigrateSourceInterface.php @@ -54,4 +54,49 @@ public function __toString(); */ public function getIds(); + /** + * The current value of the high water mark. + * + * The high water mark defines a timestamp stating the time the import was last + * run. If the mark is set, only content with a higher timestamp will be + * imported. + * + * @return int|NULL + * A Unix timestamp representing the high water mark, or NULL if no high + * water mark has been stored. + */ + public function getHighWater(); + + /** + * Get information on the property used as the high watermark. + * + * Array of 'name' & (optional) db 'alias' properties used for high watermark. + * + * @see Drupal\migrate\Plugin\migrate\source\SqlBase::initializeIterator() + * + * @return array + * The property used as the high watermark. + */ + public function getHighWaterProperty(); + + /** + * Get the name of the field used as the high watermark. + * + * The name of the field qualified with an alias if available. + * + * @see Drupal\migrate\Plugin\migrate\source\SqlBase::initializeIterator() + * + * @return string|NULL + * The name of the field for the high water mark, or NULL if not set. + */ + public function getHighWaterField(); + + /** + * Save the new high water mark. + * + * @param int $high_water + * The high water timestamp. + */ + public function saveHighWater($high_water); + } only in patch2: unchanged: --- a/core/modules/migrate/src/Plugin/migrate/source/SourcePluginBase.php +++ b/core/modules/migrate/src/Plugin/migrate/source/SourcePluginBase.php @@ -37,15 +37,6 @@ protected $migration; /** - * The name and type of the highwater property in the source. - * - * @var array - * - * @see $originalHighwater - */ - protected $highWaterProperty; - - /** * The current row from the query. * * @var \Drupal\Migrate\Row @@ -60,6 +51,20 @@ protected $currentSourceIds; /** + * Information on the property used as the high watermark. + * + * Array of 'name' & (optional) db 'alias' properties used for high watermark. + * + * @var array + */ + protected $highWaterProperty = []; + + /** + * @var \Drupal\Core\KeyValueStore\KeyValueStoreInterface + */ + protected $highWaterStorage; + + /** * The high water mark at the beginning of the import operation. * * If the source has a property for tracking changes (like Drupal ha @@ -146,10 +151,11 @@ public function __construct(array $configuration, $plugin_id, $plugin_definition $this->cacheKey = !empty($configuration['cache_key']) ? $configuration['cache_key'] : NULL; $this->trackChanges = !empty($configuration['track_changes']) ? $configuration['track_changes'] : FALSE; $this->idMap = $this->migration->getIdMap(); + $this->highWaterProperty = !empty($configuration['high_water_property']) ? $configuration['high_water_property'] : FALSE; // Pull out the current highwater mark if we have a highwater property. - if ($this->highWaterProperty = $this->migration->getHighWaterProperty()) { - $this->originalHighWater = $this->migration->getHighWater(); + if ($this->highWaterProperty) { + $this->originalHighWater = $this->getHighWater(); } // Don't allow the use of both highwater and track changes together. @@ -316,6 +322,10 @@ public function next() { continue; } + if ($this->getHighWaterProperty()) { + $this->saveHighWater($row->getSourceProperty($this->highWaterProperty['name'])); + } + // Check whether the row needs processing. // 1. This row has not been imported yet. // 2. Explicitly set to update. @@ -337,7 +347,7 @@ public function next() { * TRUE if the highwater value in the row is greater than our current value. */ protected function aboveHighwater(Row $row) { - return $this->highWaterProperty && $row->getSourceProperty($this->highWaterProperty['name']) > $this->originalHighWater; + return $this->getHighWaterProperty() && $row->getSourceProperty($this->highWaterProperty['name']) > $this->originalHighWater; } /** @@ -417,4 +427,49 @@ protected function getCache() { return $this->cache; } + /** + * Get the high water storage object. + * + * @return \Drupal\Core\KeyValueStore\KeyValueStoreInterface + * The storage object. + */ + protected function getHighWaterStorage() { + if (!isset($this->highWaterStorage)) { + $this->highWaterStorage = \Drupal::keyValue('migrate:high_water'); + } + return $this->highWaterStorage; + } + + /** + * {@inheritdoc} + */ + public function getHighWater() { + return $this->getHighWaterStorage()->get($this->migration->id()); + } + + /** + * {@inheritdoc} + */ + public function saveHighWater($high_water) { + $this->getHighWaterStorage()->set($this->migration->id(), $high_water); + } + + /** + * {@inheritdoc} + */ + public function getHighWaterProperty() { + return $this->highWaterProperty; + } + + /** + * {@inheritdoc} + */ + public function getHighWaterField() { + if (!empty($this->highWaterProperty['name'])) { + return !empty($this->highWaterProperty['alias']) ? + $this->highWaterProperty['alias'] . '.' . $this->highWaterProperty['name'] : + $this->highWaterProperty['name']; + } + return NULL; + } }