diff -u b/core/modules/migrate/src/MigrateEvents.php b/core/modules/migrate/src/MigrateEvents.php --- b/core/modules/migrate/src/MigrateEvents.php +++ b/core/modules/migrate/src/MigrateEvents.php @@ -79,2 +79,32 @@ + /** + * Name of the event fired when about to import a single item. + * + * This event allows modules to perform an action whenever a specific item + * is about to be saved by the destination plugin. The event listener method + * receives a \Drupal\migrate\MigratePreSaveEvent instance. + * + * @Event + * + * @see \Drupal\migrate\MigratePreSaveEvent + * + * @var string + */ + const PRE_SAVE = 'migrate.pre_save'; + + /** + * Name of the event fired just after a single item has been imported. + * + * This event allows modules to perform an action whenever a specific item + * has been saved by the destination plugin. The event listener method + * receives a \Drupal\migrate\MigratePostSaveEvent instance. + * + * @Event + * + * @see \Drupal\migrate\MigratePostSaveEvent + * + * @var string + */ + const POST_SAVE = 'migrate.post_save'; + } diff -u b/core/modules/migrate/src/MigrateExecutable.php b/core/modules/migrate/src/MigrateExecutable.php --- b/core/modules/migrate/src/MigrateExecutable.php +++ b/core/modules/migrate/src/MigrateExecutable.php @@ -48,44 +48,6 @@ protected $queuedMessages = array(); /** - * The options that can be set when executing the migration. - * - * Values can be set for: - * - 'limit': Sets a time limit. - * - * @var array - */ - protected $options; - - /** - * The PHP max_execution_time. - * - * @var int - */ - protected $maxExecTime; - - /** - * The ratio of the memory limit at which an operation will be interrupted. - * - * @var float - */ - protected $memoryThreshold = 0.85; - - /** - * The ratio of the time limit at which an operation will be interrupted. - * - * @var float - */ - public $timeThreshold = 0.90; - - /** - * The time limit when executing the migration. - * - * @var array - */ - public $limit = array(); - - /** * The configuration values of the source. * * @var array @@ -93,13 +55,6 @@ protected $sourceIdValues; /** - * The PHP memory_limit expressed in bytes. - * - * @var int - */ - protected $memoryLimit; - - /** * The rollback action to be saved for the current row. * * @var int @@ -157,31 +112,6 @@ $this->migration = $migration; $this->message = $message; $this->migration->getIdMap()->setMessage($message); - // Record the memory limit in bytes - $limit = trim(ini_get('memory_limit')); - if ($limit == '-1') { - $this->memoryLimit = PHP_INT_MAX; - } - else { - if (!is_numeric($limit)) { - $last = strtolower(substr($limit, -1)); - switch ($last) { - case 'g': - $limit *= 1024; - case 'm': - $limit *= 1024; - case 'k': - $limit *= 1024; - break; - default: - throw new MigrateException($this->t('Invalid PHP memory_limit !limit', - array('!limit' => $limit))); - } - } - $this->memoryLimit = $limit; - } - // Record the maximum execution time limit. - $this->maxExecTime = ini_get('max_execution_time'); } /** @@ -257,7 +187,11 @@ if ($save) { try { + $this->migration->getEventDispatcher()->dispatch(MigrateEvents::PRE_SAVE, + new MigratePreSaveEvent($this->migration, $row)); $destination_id_values = $destination->import($row, $id_map->lookupDestinationId($this->sourceIdValues)); + $this->migration->getEventDispatcher()->dispatch(MigrateEvents::POST_SAVE, + new MigratePostSaveEvent($this->migration, $row, $destination_id_values)); if ($destination_id_values) { // We do not save an idMap entry for config. if ($destination_id_values !== TRUE) { @@ -290,11 +224,8 @@ // Reset row properties. unset($sourceValues, $destinationValues); $this->sourceRowStatus = MigrateIdMapInterface::STATUS_IMPORTED; - - if (($return = $this->checkStatus()) != MigrationInterface::RESULT_COMPLETED) { - break; - } - if ($this->timeOptionExceeded()) { + if ($this->migration->getMigrationResult() == MigrationInterface::STATUS_STOPPING) { + $return = MigrationInterface::RESULT_COMPLETED; break; } try { @@ -376,40 +307,6 @@ } /** - * Tests whether we've exceeded the designated time limit. - * - * @return bool - * TRUE if the threshold is exceeded, FALSE if not. - */ - protected function timeOptionExceeded() { - // If there is no time limit, then it is not exceeded. - if (!$time_limit = $this->getTimeLimit()) { - return FALSE; - } - // Calculate if the time limit is exceeded. - $time_elapsed = $this->getTimeElapsed(); - if ($time_elapsed >= $time_limit) { - return TRUE; - } - else { - return FALSE; - } - } - - /** - * {@inheritdoc} - */ - public function getTimeLimit() { - $limit = $this->limit; - if (isset($limit['unit']) && isset($limit['value']) && ($limit['unit'] == 'seconds' || $limit['unit'] == 'second')) { - return $limit['value']; - } - else { - return NULL; - } - } - - /** * {@inheritdoc} */ public function saveMessage($message, $level = MigrationInterface::MESSAGE_ERROR) { @@ -434,133 +331,6 @@ } /** - * Checks for exceptional conditions, and display feedback. - * - * Standard top-of-loop stuff, common between rollback and import. - */ - protected function checkStatus() { - if ($this->memoryExceeded()) { - return MigrationInterface::RESULT_INCOMPLETE; - } - if ($this->maxExecTimeExceeded()) { - return MigrationInterface::RESULT_INCOMPLETE; - } - /* - * @TODO uncomment this - if ($this->getStatus() == MigrationInterface::STATUS_STOPPING) { - return MigrationBase::RESULT_STOPPED; - } - */ - - return MigrationInterface::RESULT_COMPLETED; - } - - /** - * Tests whether we've exceeded the desired memory threshold. - * - * If so, output a message. - * - * @return bool - * TRUE if the threshold is exceeded, otherwise FALSE. - */ - protected function memoryExceeded() { - $usage = $this->getMemoryUsage(); - $pct_memory = $usage / $this->memoryLimit; - if (!$threshold = $this->memoryThreshold) { - return FALSE; - } - if ($pct_memory > $threshold) { - $this->message->display( - $this->t('Memory usage is !usage (!pct% of limit !limit), reclaiming memory.', - array('!pct' => round($pct_memory*100), - '!usage' => $this->formatSize($usage), - '!limit' => $this->formatSize($this->memoryLimit))), - 'warning'); - $usage = $this->attemptMemoryReclaim(); - $pct_memory = $usage / $this->memoryLimit; - // Use a lower threshold - we don't want to be in a situation where we keep - // coming back here and trimming a tiny amount - if ($pct_memory > (0.90 * $threshold)) { - $this->message->display( - $this->t('Memory usage is now !usage (!pct% of limit !limit), not enough reclaimed, starting new batch', - array('!pct' => round($pct_memory*100), - '!usage' => $this->formatSize($usage), - '!limit' => $this->formatSize($this->memoryLimit))), - 'warning'); - return TRUE; - } - else { - $this->message->display( - $this->t('Memory usage is now !usage (!pct% of limit !limit), reclaimed enough, continuing', - array('!pct' => round($pct_memory*100), - '!usage' => $this->formatSize($usage), - '!limit' => $this->formatSize($this->memoryLimit))), - 'warning'); - return FALSE; - } - } - else { - return FALSE; - } - } - - /** - * Returns the memory usage so far. - * - * @return int - * The memory usage. - */ - protected function getMemoryUsage() { - return memory_get_usage(); - } - - /** - * Tries to reclaim memory. - * - * @return int - * The memory usage after reclaim. - */ - protected function attemptMemoryReclaim() { - // First, try resetting Drupal's static storage - this frequently releases - // plenty of memory to continue. - drupal_static_reset(); - // @TODO: explore resetting the container. - return memory_get_usage(); - } - - /** - * Generates a string representation for the given byte count. - * - * @param int $size - * A size in bytes. - * - * @return string - * A translated string representation of the size. - */ - protected function formatSize($size) { - return format_size($size); - } - - /** - * Tests whether we're approaching the PHP maximum execution time limit. - * - * @return bool - * TRUE if the threshold is exceeded, FALSE if not. - */ - protected function maxExecTimeExceeded() { - return $this->maxExecTime && (($this->getTimeElapsed() / $this->maxExecTime) > $this->timeThreshold); - } - - /** - * Returns the time elapsed. - * - * This allows a test to set a fake elapsed time. - */ - protected function getTimeElapsed() { - return time() - REQUEST_TIME; - } - - /** * Takes an Exception object and both saves and displays it. * * Pulls in additional information on the location triggering the exception. diff -u b/core/modules/migrate/src/MigratePostImportEvent.php b/core/modules/migrate/src/MigratePostImportEvent.php --- b/core/modules/migrate/src/MigratePostImportEvent.php +++ b/core/modules/migrate/src/MigratePostImportEvent.php @@ -16,7 +16,7 @@ class MigratePostImportEvent extends Event { /** - * Map object. + * Migration object. * * @var \Drupal\migrate\Entity\MigrationInterface */ diff -u b/core/modules/migrate/src/MigratePreImportEvent.php b/core/modules/migrate/src/MigratePreImportEvent.php --- b/core/modules/migrate/src/MigratePreImportEvent.php +++ b/core/modules/migrate/src/MigratePreImportEvent.php @@ -16,7 +16,7 @@ class MigratePreImportEvent extends Event { /** - * Map object. + * Migration object. * * @var \Drupal\migrate\Entity\MigrationInterface */ diff -u b/core/modules/migrate/tests/src/Unit/MigrateExecutableTest.php b/core/modules/migrate/tests/src/Unit/MigrateExecutableTest.php --- b/core/modules/migrate/tests/src/Unit/MigrateExecutableTest.php +++ b/core/modules/migrate/tests/src/Unit/MigrateExecutableTest.php @@ -53,8 +53,6 @@ $this->executable = new TestMigrateExecutable($this->migration, $this->message); $this->executable->setStringTranslation($this->getStringTranslationStub()); - $this->executable->setTimeThreshold(0.1); - $this->executable->limit = array('unit' => 'second', 'value' => 1); } /** @@ -337,44 +335,6 @@ } /** - * Tests time limit option method. - */ - public function testTimeOptionExceeded() { - // Assert time limit of one second (test configuration default) is exceeded. - $this->executable->setTimeElapsed(1); - $this->assertTrue($this->executable->timeOptionExceeded()); - // Assert time limit not exceeded. - $this->executable->limit = array('unit' => 'seconds', 'value' => (int) $_SERVER['REQUEST_TIME'] - 3600); - $this->assertFalse($this->executable->timeOptionExceeded()); - // Assert no time limit. - $this->executable->limit = array(); - $this->assertFalse($this->executable->timeOptionExceeded()); - } - - /** - * Tests get time limit method. - */ - public function testGetTimeLimit() { - // Assert time limit has a unit of one second (test configuration default). - $limit = $this->executable->limit; - $this->assertArrayHasKey('unit', $limit); - $this->assertSame('second', $limit['unit']); - $this->assertSame($limit['value'], $this->executable->getTimeLimit()); - // Assert time limit has a unit of multiple seconds. - $this->executable->limit = array('unit' => 'seconds', 'value' => 30); - $limit = $this->executable->limit; - $this->assertArrayHasKey('unit', $limit); - $this->assertSame('seconds', $limit['unit']); - $this->assertSame($limit['value'], $this->executable->getTimeLimit()); - // Assert no time limit. - $this->executable->limit = array(); - $limit = $this->executable->limit; - $this->assertArrayNotHasKey('unit', $limit); - $this->assertArrayNotHasKey('value', $limit); - $this->assertNull($this->executable->getTimeLimit()); - } - - /** * Tests saving of queued messages. */ public function testSaveQueuedMessages() { @@ -426,22 +386,6 @@ } /** - * Tests maximum execution time (max_execution_time) of an import. - */ - public function testMaxExecTimeExceeded() { - // Assert no max_execution_time value. - $this->executable->setMaxExecTime(0); - $this->assertFalse($this->executable->maxExecTimeExceeded()); - // Assert default max_execution_time value does not exceed. - $this->executable->setMaxExecTime(30); - $this->assertFalse($this->executable->maxExecTimeExceeded()); - // Assert max_execution_time value is exceeded. - $this->executable->setMaxExecTime(1); - $this->executable->setTimeElapsed(2); - $this->assertTrue($this->executable->maxExecTimeExceeded()); - } - - /** * Tests the processRow method. */ public function testProcessRow() { diff -u b/core/modules/migrate/tests/src/Unit/MigrateSqlIdMapEnsureTablesTest.php b/core/modules/migrate/tests/src/Unit/MigrateSqlIdMapEnsureTablesTest.php --- b/core/modules/migrate/tests/src/Unit/MigrateSqlIdMapEnsureTablesTest.php +++ b/core/modules/migrate/tests/src/Unit/MigrateSqlIdMapEnsureTablesTest.php @@ -194,10 +194,6 @@ $migration->expects($this->any()) ->method('getSourcePlugin') ->willReturn($plugin); - $event_dispatcher = $this->getMock('Symfony\Component\EventDispatcher\EventDispatcherInterface'); - $migration->expects($this->any()) - ->method('getEventDispatcher') - ->willReturn($event_dispatcher); $plugin = $this->getMock('Drupal\migrate\Plugin\MigrateSourceInterface'); $plugin->expects($this->any()) ->method('getIds') diff -u b/core/modules/migrate/tests/src/Unit/TestMigrateExecutable.php b/core/modules/migrate/tests/src/Unit/TestMigrateExecutable.php --- b/core/modules/migrate/tests/src/Unit/TestMigrateExecutable.php +++ b/core/modules/migrate/tests/src/Unit/TestMigrateExecutable.php @@ -16,27 +16,6 @@ class TestMigrateExecutable extends MigrateExecutable { /** - * The (fake) number of seconds elapsed since the start of the test. - * - * @var int - */ - protected $timeElapsed; - - /** - * The fake memory usage in bytes. - * - * @var int - */ - protected $memoryUsage; - - /** - * The cleared memory usage. - * - * @var int - */ - protected $clearedMemoryUsage; - - /** * Sets the string translation service. * * @param \Drupal\Core\StringTranslation\TranslationInterface $string_translation @@ -47,46 +26,6 @@ } /** - * Allows access to protected timeOptionExceeded method. - * - * @return bool - * A threshold exceeded value. - */ - public function timeOptionExceeded() { - return parent::timeOptionExceeded(); - } - - /** - * Allows access to set protected maxExecTime property. - * - * @param int $max_exec_time - * The value to set. - */ - public function setMaxExecTime($max_exec_time) { - $this->maxExecTime = $max_exec_time; - } - - /** - * Allows access to protected maxExecTime property. - * - * @return int - * The value of the protected property. - */ - public function getMaxExecTime() { - return $this->maxExecTime; - } - - /** - * Allows access to protected maxExecTimeExceeded method. - * - * @return bool - * The threshold exceeded value. - */ - public function maxExecTimeExceeded() { - return parent::maxExecTimeExceeded(); - } - - /** * Allows access to set protected source property. * * @param \Drupal\migrate\Plugin\MigrateSourceInterface $source @@ -107,23 +46,6 @@ } /** - * Allows setting a fake elapsed time. - * - * @param int $time - * The time in seconds. - */ - public function setTimeElapsed($time) { - $this->timeElapsed = $time; - } - - /** - * {@inheritdoc} - */ - public function getTimeElapsed() { - return $this->timeElapsed; - } - - /** * {@inheritdoc} */ public function handleException(\Exception $exception, $save = TRUE) { @@ -135,73 +57,6 @@ } /** - * Allows access to the protected memoryExceeded method. - * - * @return bool - * The memoryExceeded value. - */ - public function memoryExceeded() { - return parent::memoryExceeded(); - } - - /** - * {@inheritdoc} - */ - protected function attemptMemoryReclaim() { - return $this->clearedMemoryUsage; - } - - /** - * {@inheritdoc} - */ - protected function getMemoryUsage() { - return $this->memoryUsage; - } - - /** - * Sets the fake memory usage. - * - * @param int $memory_usage - * The fake memory usage value. - * @param int $cleared_memory_usage - * (optional) The fake cleared memory value. - */ - public function setMemoryUsage($memory_usage, $cleared_memory_usage = NULL) { - $this->memoryUsage = $memory_usage; - $this->clearedMemoryUsage = $cleared_memory_usage; - } - - /** - * Sets the memory limit. - * - * @param int $memory_limit - * The memory limit. - */ - public function setMemoryLimit($memory_limit) { - $this->memoryLimit = $memory_limit; - } - - /** - * Sets the memory threshold. - * - * @param float $threshold - * The new threshold. - */ - public function setMemoryThreshold($threshold) { - $this->memoryThreshold = $threshold; - } - - /** - * Sets the time threshold. - * - * @param float $threshold - * The new threshold. - */ - public function setTimeThreshold($threshold) { - $this->timeThreshold = $threshold; - } - - /** * {@inheritdoc} */ protected function formatSize($size) { only in patch2: unchanged: --- a/core/modules/migrate/src/MigrateExecutableInterface.php +++ b/core/modules/migrate/src/MigrateExecutableInterface.php @@ -37,14 +37,6 @@ public function import(); public function processRow(Row $row, array $process = NULL, $value = NULL); /** - * Returns the time limit. - * - * @return null|int - * The time limit, NULL if no limit or if the units were not in seconds. - */ - public function getTimeLimit(); - - /** * Passes messages through to the map class. * * @param string $message only in patch2: unchanged: --- /dev/null +++ b/core/modules/migrate/src/MigratePostSaveEvent.php @@ -0,0 +1,85 @@ +migration = $migration; + $this->row = $row; + $this->destinationIdValues = $destination_id_values; + } + + /** + * Gets migration object. + * + * @return \Drupal\migrate\Entity\MigrationInterface + * The migration object being imported. + */ + public function getMigration() { + return $this->migration; + } + + /** + * Gets row object. + * + * @return \Drupal\migrate\Row + * The row object just imported. + */ + public function getRow() { + return $this->migration; + } + + /** + * Gets destination ID values. + * + * @return array + * The destination ID as an array. + */ + public function getDestinationIdValues() { + return $this->destinationIdValues; + } + +} only in patch2: unchanged: --- /dev/null +++ b/core/modules/migrate/src/MigratePreSaveEvent.php @@ -0,0 +1,63 @@ +migration = $migration; + $this->row = $row; + } + + /** + * Gets migration object. + * + * @return \Drupal\migrate\Entity\MigrationInterface + * The migration object being imported. + */ + public function getMigration() { + return $this->migration; + } + + /** + * Gets row object. + * + * @return \Drupal\migrate\Row + * The row object about to be imported. + */ + public function getRow() { + return $this->migration; + } + +} only in patch2: unchanged: --- a/core/modules/migrate/tests/src/Unit/MigrateExecutableMemoryExceededTest.php +++ /dev/null @@ -1,120 +0,0 @@ - 'test', - ); - - /** - * php.init memory_limit value. - */ - protected $memoryLimit = 10000000; - - /** - * {@inheritdoc} - */ - protected function setUp() { - parent::setUp(); - $this->migration = $this->getMigration(); - $this->message = $this->getMock('Drupal\migrate\MigrateMessageInterface'); - - $this->executable = new TestMigrateExecutable($this->migration, $this->message); - $this->executable->setStringTranslation($this->getStringTranslationStub()); - } - - /** - * Runs the actual test. - * - * @param string $message - * The second message to assert. - * @param bool $memory_exceeded - * Whether to test the memory exceeded case. - * @param int $memory_usage_first - * (optional) The first memory usage value. - * @param int $memory_usage_second - * (optional) The fake amount of memory usage reported after memory reclaim. - * @param int $memory_limit - * (optional) The memory limit. - */ - protected function runMemoryExceededTest($message, $memory_exceeded, $memory_usage_first = NULL, $memory_usage_second = NULL, $memory_limit = NULL) { - $this->executable->setMemoryLimit($memory_limit ?: $this->memoryLimit); - $this->executable->setMemoryUsage($memory_usage_first ?: $this->memoryLimit, $memory_usage_second ?: $this->memoryLimit); - $this->executable->setMemoryThreshold(0.85); - if ($message) { - $this->executable->message->expects($this->at(0)) - ->method('display') - ->with($this->stringContains('reclaiming memory')); - $this->executable->message->expects($this->at(1)) - ->method('display') - ->with($this->stringContains($message)); - } - else { - $this->executable->message->expects($this->never()) - ->method($this->anything()); - } - $result = $this->executable->memoryExceeded(); - $this->assertEquals($memory_exceeded, $result); - } - - /** - * Tests memoryExceeded method when a new batch is needed. - */ - public function testMemoryExceededNewBatch() { - // First case try reset and then start new batch. - $this->runMemoryExceededTest('starting new batch', TRUE); - } - - /** - * Tests memoryExceeded method when enough is cleared. - */ - public function testMemoryExceededClearedEnough() { - $this->runMemoryExceededTest('reclaimed enough', FALSE, $this->memoryLimit, $this->memoryLimit * 0.75); - } - - /** - * Tests memoryExceeded when memory usage is not exceeded. - */ - public function testMemoryNotExceeded() { - $this->runMemoryExceededTest('', FALSE, floor($this->memoryLimit * 0.85) - 1); - } - -} only in patch2: unchanged: --- a/core/modules/migrate/tests/src/Unit/MigrateTestCase.php +++ b/core/modules/migrate/tests/src/Unit/MigrateTestCase.php @@ -51,6 +51,10 @@ protected function getMigration() { $migration->expects($this->any()) ->method('id') ->will($this->returnValue($configuration['id'])); + $event_dispatcher = $this->getMock('Symfony\Component\EventDispatcher\EventDispatcherInterface'); + $migration->expects($this->any()) + ->method('getEventDispatcher') + ->willReturn($event_dispatcher); return $migration; }