diff -u b/core/modules/migrate/migrate.services.yml b/core/modules/migrate/migrate.services.yml --- b/core/modules/migrate/migrate.services.yml +++ b/core/modules/migrate/migrate.services.yml @@ -35 +35,6 @@ - arguments: ['0.9', '0.85'] + arguments: ['@event_dispatcher', '0.9', '0.85'] + migrate.memory_limit_exceeded: + class: Drupal\migrate\EventSubscriber\MemoryLimitExceeded + tags: + - { name: 'event_subscriber } + arguments: ['@entity_type.manager'] diff -u b/core/modules/migrate/src/MemoryManager.php b/core/modules/migrate/src/MemoryManager.php --- b/core/modules/migrate/src/MemoryManager.php +++ b/core/modules/migrate/src/MemoryManager.php @@ -3,15 +3,15 @@ namespace Drupal\migrate; use Drupal\Component\Utility\Bytes; -use Drupal\Core\StringTranslation\StringTranslationTrait; +use Drupal\migrate\Event\MigrateEvents; +use Drupal\migrate\Event\MigrateMemoryLimitEvent; +use Symfony\Component\EventDispatcher\EventDispatcherInterface; /** * The memory manager for migrations. */ class MemoryManager implements MemoryManagerInterface { - use StringTranslationTrait; - /** * The ratio of the memory limit which will trigger a failed reclaim. * @@ -33,24 +33,19 @@ */ protected $memoryLimit; - /** - * Migration message service. - * - * @var \Drupal\migrate\MigrateMessageInterface - */ - protected $message; /** - * The entity type manager. + * The event dispatcher. * - * @var \Drupal\Core\Entity\EntityTypeManagerInterface + * @var \Symfony\Component\EventDispatcher\EventDispatcherInterface */ - protected $entityTypeManager; + protected $dispatcher; /** * MemoryManager constructor. */ - public function __construct($reclaim_threshold, $max_threshold, $memory_limit = NULL) { + public function __construct(EventDispatcherInterface $dispatcher, $reclaim_threshold, $max_threshold, $memory_limit = NULL) { + $this->dispatcher = $dispatcher; $this->memoryReclaimThreshold = $reclaim_threshold; $this->memoryThreshold = $max_threshold; // Record the memory limit in bytes. @@ -67,45 +62,24 @@ $this->memoryLimit = Bytes::toInt($limit); } } - $this->message = new MigrateMessage(); } /** * {@inheritdoc} */ public function ensureMemory() { - $ratio_multiplier = 100; - if ($this->isLimitExceeded()) { - $this->getMigrateMessage()->display($this->t( - 'Memory usage is @usage (@pct% of limit @limit), reclaiming memory.', - [ - '@pct' => round($this->getUsageRatio() * $ratio_multiplier), - '@usage' => $this->formatSize($this->getUsageInBytes()), - '@limit' => $this->formatSize($this->getLimit()), - ] - ), 'warning'); - - // Check the lower reclaim threshold to ensure we reclaimed enough to continue. - if ($this->reclaim()->isLimitExceeded($this->memoryReclaimThreshold)) { - $this->getMigrateMessage()->display($this->t( - 'Memory usage is now @usage (@pct% of limit @limit), not enough reclaimed, starting new batch', - [ - '@pct' => round($this->getUsageRatio() * $ratio_multiplier), - '@usage' => $this->formatSize($this->getUsageInBytes()), - '@limit' => $this->formatSize($this->getLimit()), - ] - ), 'warning'); + $event = new MigrateMemoryLimitEvent($this->getUsageRatio(), $this->getUsageInBytes(), $this->getLimit(), 'pre reclaimed'); + $this->dispatcher->dispatch(MigrateEvents::MEMORY_LIMIT, $event); + // Re-check the reclaim threshold to ensure we reclaimed enough to + // continue. + if ($this->isLimitExceeded($this->memoryReclaimThreshold)) { + $event = new MigrateMemoryLimitEvent($this->getUsageRatio(), $this->getUsageInBytes(), $this->getLimit(), 'still exceeded'); + $this->dispatcher->dispatch(MigrateEvents::MEMORY_LIMIT, $event); return FALSE; } - $this->getMigrateMessage()->display($this->t( - 'Memory usage is now @usage (@pct% of limit @limit), reclaimed enough, continuing', - [ - '@pct' => round($this->getUsageRatio() * $ratio_multiplier), - '@usage' => $this->formatSize($this->getUsageInBytes()), - '@limit' => $this->formatSize($this->getLimit()), - ] - ), 'warning'); + $event = new MigrateMemoryLimitEvent($this->getUsageRatio(), $this->getUsageInBytes(), $this->getLimit(), 'reduced enough to continue'); + $this->dispatcher->dispatch(MigrateEvents::MEMORY_LIMIT, $event); } return TRUE; } @@ -140,31 +114,6 @@ } /** - * Tries to reclaim memory. - * - * @return $this - * The called object for chaining. - */ - protected function reclaim() { - // First, try resetting Drupal's static storage - this frequently releases - // plenty of memory to continue. - drupal_static_reset(); - - // Entity storage can blow up with caches so clear them out. - foreach ($this->getEntityTypeManager() - ->getDefinitions() as $id => $definition) { - $this->getEntityTypeManager()->getStorage($id)->resetCache(); - } - - // @TODO: explore resetting the container. - - // Run garbage collector to further reduce memory. - gc_collect_cycles(); - - return $this; - } - - /** * Checks if the memory limit has been exceeded. * * @param int $multiplier @@ -184,42 +133,2 @@ - /** - * Generates a string representation for the given byte count. - * - * @param int $size - * A size in bytes. - * - * @return string - * A translated string representation of the size. - */ - protected function formatSize($size) { - return format_size($size); - } - - /** - * Get the entity type manager. - * - * @return \Drupal\Core\Entity\EntityTypeManagerInterface - * The entity type manager. - */ - protected function getEntityTypeManager() { - if (!$this->entityTypeManager) { - $this->entityTypeManager = \Drupal::entityTypeManager(); - } - return $this->entityTypeManager; - } - - /** - * {@inheritdoc} - */ - public function setMigrateMessage(MigrateMessageInterface $message) { - $this->message = $message; - } - - protected function getMigrateMessage() { - if (!$this->message) { - $this->message = new MigrateMessage(); - } - return $this->message; - } - } diff -u b/core/modules/migrate/src/MemoryManagerInterface.php b/core/modules/migrate/src/MemoryManagerInterface.php --- b/core/modules/migrate/src/MemoryManagerInterface.php +++ b/core/modules/migrate/src/MemoryManagerInterface.php @@ -20,10 +20,2 @@ - /** - * Set the migrate message service. - * - * @param \Drupal\migrate\MigrateMessageInterface $message - * The migrate message service. - */ - public function setMigrateMessage(MigrateMessageInterface $message); - } only in patch2: unchanged: --- a/core/modules/migrate/src/Event/MigrateEvents.php +++ b/core/modules/migrate/src/Event/MigrateEvents.php @@ -181,4 +181,19 @@ */ const IDMAP_MESSAGE = 'migrate.idmap_message'; + /** + * Name of the event fired when the memory limit is reached. + * + * This event allows modules to perform and action when the limit is reached. + * The event lister method receives a + * \Drupal\migrate\Event\MigrateMemoryLimitEvent + * + * @Event + * + * @see \Drupal\migrate\Event\MigrateMemoryLimitEvent + * + * @var string + */ + const MEMORY_LIMIT = 'migration.memory_limit'; + } only in patch2: unchanged: --- /dev/null +++ b/core/modules/migrate/src/Event/MigrateMemoryLimitEvent.php @@ -0,0 +1,98 @@ +usageRatio = $usage_ratio; + $this->usageInBytes = $usage_in_bytes; + $this->limit = $limit; + $this->phase = $phase; + } + + /** + * Get the memory usage ratio. + * + * @return float|int + */ + public function getUsageRatio() { + return $this->usageRatio; + } + + /** + * Get the memory usage in bytes. + * + * @return int + */ + public function getUsageInBytes() { + return $this->usageInBytes; + } + + /** + * Get the memory limit. + * + * @return int + */ + public function getLimit() { + return $this->limit; + } + + /** + * The phase of memory reclamation. + * + * @return string + */ + public function getPhase() { + return $this->phase; + } + +} only in patch2: unchanged: --- /dev/null +++ b/core/modules/migrate/src/EventSubscriber/MemoryLimitExceeded.php @@ -0,0 +1,124 @@ +entityTypeManager = $entity_type_manager; + $this->message = new MigrateMessage(); + } + + /** + * {@inheritdoc} + */ + public static function getSubscribedEvents() { + $events[MigrateEvents::MEMORY_LIMIT][] = 'reclaim'; + $events[MigrateEvents::MEMORY_LIMIT][] = 'notify'; + return $events; + } + + /** + * Tries to reclaim memory. + */ + public function reclaim(MigrateMemoryLimitEvent $event) { + if ($event->getPhase() != 'pre reclaimed') { + return; + } + // First, try resetting Drupal's static storage - this frequently releases + // plenty of memory to continue. + drupal_static_reset(); + + // Entity storage can blow up with caches so clear them out. + foreach ($this->entityTypeManager->getDefinitions() as $id => $definition) { + $this->entityTypeManager->getStorage($id)->resetCache(); + } + + // @TODO: explore resetting the container. + + // Run garbage collector to further reduce memory. + gc_collect_cycles(); + } + + public function notify(MigrateMemoryLimitEvent $event) { + $ratio_multiplier = 100; + switch ($event->getPhase()) { + case 'pre reclaimed': + $this->message->display($this->t( + 'Memory usage is @usage (@pct% of limit @limit), reclaiming memory.', + [ + '@pct' => round($event->getUsageRatio() * $ratio_multiplier), + '@usage' => $this->formatSize($event->getUsageInBytes()), + '@limit' => $this->formatSize($event->getLimit()), + ] + ), 'warning'); + break; + + case 'still exceeded': + $this->message->display($this->t( + 'Memory usage is now @usage (@pct% of limit @limit), not enough reclaimed, starting new batch', + [ + '@pct' => round($event->getUsageRatio() * $ratio_multiplier), + '@usage' => $this->formatSize($event->getUsageInBytes()), + '@limit' => $this->formatSize($event->getLimit()), + ] + ), 'warning'); + break; + + case '': + $this->message->display($this->t( + 'Memory usage is now @usage (@pct% of limit @limit), reclaimed enough, continuing', + [ + '@pct' => round($event->getUsageRatio() * $ratio_multiplier), + '@usage' => $this->formatSize($event->getUsageInBytes()), + '@limit' => $this->formatSize($event->getLimit()), + ] + )); + break; + } + } + + /** + * Generates a string representation for the given byte count. + * + * @param int $size + * A size in bytes. + * + * @return string + * A translated string representation of the size. + */ + protected function formatSize($size) { + return format_size($size); + } + +}