diff --git a/default_content_deploy.services.yml b/default_content_deploy.services.yml index 4f0b3f9..11d69f0 100644 --- a/default_content_deploy.services.yml +++ b/default_content_deploy.services.yml @@ -4,7 +4,7 @@ services: arguments: ['@serializer', '@entity_type.manager', '@hal.link_manager', '@account_switcher', '@default_content_deploy.manager', '@entity.repository', '@cache.default', '@default_content_deploy.exporter', '@database', '@event_dispatcher'] default_content_deploy.exporter: class: Drupal\default_content_deploy\Exporter - arguments: ['@database', '@default_content_deploy.manager', '@entity_type.manager', '@serializer', '@account_switcher', '@file_system', '@hal.link_manager', '@event_dispatcher', '@module_handler'] + arguments: ['@database', '@default_content_deploy.manager', '@entity_type.manager', '@serializer', '@account_switcher', '@file_system', '@hal.link_manager', '@event_dispatcher', '@module_handler', '@config.factory', '@language_manager', '@entity.repository'] default_content_deploy.manager: class: Drupal\default_content_deploy\DeployManager arguments: ['@entity_type.manager', '@config.factory', '@settings', '@file_system', '@request_stack'] diff --git a/src/Commands/DefaultContentDeployCommands.php b/src/Commands/DefaultContentDeployCommands.php index 0508acd..f91d068 100644 --- a/src/Commands/DefaultContentDeployCommands.php +++ b/src/Commands/DefaultContentDeployCommands.php @@ -289,12 +289,8 @@ class DefaultContentDeployCommands extends DrushCommands { $this->importer->setFolder($options['folder']); } - $this->importer->prepareForImport(); - $this->displayImportResult(); - - if (!$this->isAllSkip() && $this->io()->confirm(dt('Do you really want to continue?'))) { + if ($this->io()->confirm(dt('Do you really want to continue?'))) { $this->importer->import(); - $this->io()->success(dt('Content has been imported.')); } } @@ -349,52 +345,6 @@ class DefaultContentDeployCommands extends DrushCommands { } } - /** - * Display info before/after import. - */ - private function displayImportResult() { - $result = $this->importer->getResult(); - $array_column = array_column($result, 'status'); - $count = array_count_values($array_column); - - if ($this->output()->isVerbose()) { - $table = new Table($this->output()); - $table->setHeaders(['Action', 'Entity Type', 'UUID']); - foreach ($result as $uuid => $data) { - $table->addRow([$data['status'], $data['entity_type_id'], $uuid]); - } - $table->render(); - $this->output()->writeln(dt('Summary:')); - } - - $this->output()->writeln(dt('- created: @count', [ - '@count' => isset($count['create']) ? $count['create'] : 0, - ])); - - $this->output()->writeln(dt('- updated: @count', [ - '@count' => isset($count['update']) ? $count['update'] : 0, - ])); - - $this->output()->writeln(dt('- skipped: @count', [ - '@count' => isset($count['skip']) ? $count['skip'] : 0, - ])); - } - - /** - * Is update or create not exist. - * - * @return bool - */ - private function isAllSkip() { - $result = $this->importer->getResult(); - $array_column = array_column($result, 'status'); - $count = array_count_values($array_column); - $create_count = isset($count['create']) ? $count['create'] : 0; - $update_count = isset($count['update']) ? $count['update'] : 0; - - return ($create_count == 0 && $update_count == 0); - } - /** * Helper for processing array drush options. * diff --git a/src/Exporter.php b/src/Exporter.php index 08ef077..2a8c85a 100644 --- a/src/Exporter.php +++ b/src/Exporter.php @@ -3,12 +3,17 @@ namespace Drupal\default_content_deploy; use Drupal\Component\EventDispatcher\ContainerAwareEventDispatcher; +use Drupal\Component\Utility\Html; +use Drupal\Core\Config\ConfigFactoryInterface; use Drupal\Core\Database\Connection; +use Drupal\Core\DependencyInjection\DependencySerializationTrait; use Drupal\Core\Entity\ContentEntityInterface; use Drupal\Core\Entity\EntityChangedInterface; +use Drupal\Core\Entity\EntityRepositoryInterface; use Drupal\Core\Entity\EntityTypeManagerInterface; use Drupal\Core\Extension\ModuleHandlerInterface; use Drupal\Core\File\FileSystemInterface; +use Drupal\Core\Language\LanguageManagerInterface; use Drupal\Core\Session\AccountSwitcherInterface; use Drupal\default_content_deploy\Event\PreSerializeEvent; use Drupal\hal\LinkManager\LinkManagerInterface; @@ -22,6 +27,29 @@ use Drupal\layout_builder\SectionComponent; */ class Exporter { + use DependencySerializationTrait; + + /** + * The config factory. + * + * @var \Drupal\Core\Config\ConfigFactoryInterface + */ + protected $config; + + /** + * The language manager service. + * + * @var \Drupal\Core\Language\LanguageManagerInterface + */ + protected $languageManager; + + /** + * The entity repository. + * + * @var \Drupal\Core\Entity\EntityRepositoryInterface + */ + protected $entityRepository; + /** * DCD Manager. * @@ -144,8 +172,7 @@ class Exporter { */ protected $moduleHandler; - /** - * Exporter constructor. + /* Exporter constructor. * * @param \Drupal\Core\Database\Connection $database * DB connection. @@ -165,8 +192,14 @@ class Exporter { * The event dispatcher. * @param \Drupal\Core\Extension\ModuleHandlerInterface $module_handler * The module handler service. - */ - public function __construct(Connection $database, DeployManager $deploy_manager, EntityTypeManagerInterface $entityTypeManager, Serializer $serializer, AccountSwitcherInterface $account_switcher, FileSystemInterface $file_system, LinkManagerInterface $link_manager, ContainerAwareEventDispatcher $eventDispatcher, ModuleHandlerInterface $module_handler) { + * @param \Drupal\Core\Config\ConfigFactoryInterface $config + * The config factory. + * @param \Drupal\Core\Language\LanguageManagerInterface $language_manager + * The language manager service. + * @param \Drupal\Core\Entity\EntityRepositoryInterface $entity_repository + * The entity repository. + */ + public function __construct(Connection $database, DeployManager $deploy_manager, EntityTypeManagerInterface $entityTypeManager, Serializer $serializer, AccountSwitcherInterface $account_switcher, FileSystemInterface $file_system, LinkManagerInterface $link_manager, ContainerAwareEventDispatcher $eventDispatcher, ModuleHandlerInterface $module_handler, ConfigFactoryInterface $config, LanguageManagerInterface $language_manager, EntityRepositoryInterface $entity_repository) { $this->database = $database; $this->entityTypeManager = $entityTypeManager; $this->serializer = $serializer; @@ -176,6 +209,9 @@ class Exporter { $this->linkManager = $link_manager; $this->eventDispatcher = $eventDispatcher; $this->moduleHandler = $module_handler; + $this->config = $config; + $this->languageManager = $language_manager; + $this->entityRepository = $entity_repository; } /** @@ -347,32 +383,31 @@ class Exporter { public function export() { switch ($this->mode) { case 'default': - $this->prepareToExport(); + $this->exportBatch(); break; case 'reference': - $this->prepareToExportWithReference(); + $this->exportBatch(TRUE); break; case 'all': - $this->prepareToExportAllContent(); + $this->exportAllBatch(); break; } - // Edit and export all entities to folder. - $this->editEntityData(); - $this->writeConfigsToFolder(); - return $this; } /** * Prepare content to export. * + * @param boolean $with_references + * Indicates if export should consider referenced entities. + * * @throws \Drupal\Component\Plugin\Exception\InvalidPluginDefinitionException * @throws \Drupal\Component\Plugin\Exception\PluginNotFoundException */ - private function prepareToExport() { + private function exportBatch($with_references = FALSE) { $entity_type = $this->entityTypeId; $exported_entity_ids = $this->getEntityIdsForExport(); @@ -380,33 +415,113 @@ class Exporter { $this->fileSystem->deleteRecursive($this->getFolder()); } + $export_type = $with_references ? "exportBatchWithReferences" : "exportBatchDefault"; + foreach ($exported_entity_ids as $entity_id) { - /** @var \Drupal\Core\Entity\ContentEntityInterface $entity */ - $entity = $this->entityTypeManager->getStorage($entity_type)->load($entity_id); - $exported_entity = $this->getSerializedContent($entity); - $this->addExportedEntity($exported_entity); + $operations[] = [ + [$this, $export_type], + [$entity_type, $entity_id], + ]; + } + + // Set up batch information. + $batch = [ + 'title' => t('Exporting content'), + 'operations' => $operations, + 'finished' => [$this, 'exportFinished'], + ]; + + batch_set($batch); + + if (PHP_SAPI === 'cli') { + drush_backend_batch_process(); } } /** - * Prepare content with reference to export. + * Prepares and exports a single entity to a JSON file. * - * @throws \Drupal\Component\Plugin\Exception\InvalidPluginDefinitionException - * @throws \Drupal\Component\Plugin\Exception\PluginNotFoundException - */ - private function prepareToExportWithReference() { - $entity_type = $this->entityTypeId; - $exported_entity_ids = $this->getEntityIdsForExport(); - - if ($this->forceUpdate) { - $this->fileSystem->deleteRecursive($this->getFolder()); + * @param string $entity_type + * The type of the entity being exported. + * @param int $entity_id + * The ID of the entity being exported. + * @param array $context + * The batch context. + */ + public function exportBatchDefault($entity_type, $entity_id, &$context) { + // Prepare export of entity. + $entity = $this->entityTypeManager->getStorage($entity_type)->load($entity_id); + $exported_entity = $this->getSerializedContent($entity); + + // Remove or add a new fields to serialize entities data. + $entity_array = $this->serializer->decode($exported_entity, 'hal_json'); + $entity_type_object = $this->entityTypeManager->getDefinition($entity_type); + $id_key = $entity_type_object->getKey('id'); + $entity_id = $entity_array[$id_key][0]['value']; + $entity = $this->entityTypeManager->getStorage($entity_type)->load($entity_id); + + unset($entity_array[$entity_type_object->getKey('revision')]); + + if ($entity_type === 'user') { + $entity_array['pass'][0]['value'] = $entity->getPassword(); } - foreach ($exported_entity_ids as $entity_id) { - /** @var \Drupal\Core\Entity\ContentEntityInterface $entity */ - $entity = $this->entityTypeManager->getStorage($entity_type)->load($entity_id); + $data = $this->serializer->serialize($entity_array, 'hal_json', [ + 'json_encode_options' => JSON_PRETTY_PRINT + ]); + + // Write serialized entity to JSON file. + $uuid = $entity->get('uuid')->value; + $entity_type_folder = "{$this->getFolder()}/{$entity_type}"; + $this->fileSystem->prepareDirectory($entity_type_folder, FileSystemInterface::CREATE_DIRECTORY); + + file_put_contents("{$entity_type_folder}/{$uuid}.json", $data); + } + + /** + * Prepares and exports a single entity to a JSON file with references. + * + * @param string $entity_type + * The type of the entity being exported. + * @param int $entity_id + * The ID of the entity being exported. + * @param array $context + * The batch context. + */ + public function exportBatchWithReferences($entity_type, $entity_id, &$context) { + // Get referenced entities. + $entity = $this->entityTypeManager->getStorage($entity_type)->load($entity_id); + + $exported_entities = []; + if ($entity instanceof ContentEntityInterface) { $exported_entities = $this->getSerializedContentWithReferences($entity); - $this->addExportedEntity($exported_entities); + } + + foreach ($exported_entities as $entity_type => $exported_entity_array) { + foreach ($exported_entity_array as $uuid => $exported_entity) { + // Remove or add a new fields to serialize entities data. + $entity_array = $this->serializer->decode($exported_entity, 'hal_json'); + $entity_type_object = $this->entityTypeManager->getDefinition($entity_type); + $id_key = $entity_type_object->getKey('id'); + $entity_id = $entity_array[$id_key][0]['value']; + $entity = $this->entityTypeManager->getStorage($entity_type)->load($entity_id); + + unset($entity_array[$entity_type_object->getKey('revision')]); + + if ($entity_type === 'user') { + $entity_array['pass'][0]['value'] = $entity->getPassword(); + } + + $data = $this->serializer->serialize($entity_array, 'hal_json', [ + 'json_encode_options' => JSON_PRETTY_PRINT + ]); + + // Write serialized entity to JSON file. + $entity_type_folder = "{$this->getFolder()}/{$entity_type}"; + $this->fileSystem->prepareDirectory($entity_type_folder, FileSystemInterface::CREATE_DIRECTORY); + + file_put_contents("{$entity_type_folder}/{$uuid}.json", $data); + } } } @@ -416,7 +531,7 @@ class Exporter { * @throws \Drupal\Component\Plugin\Exception\InvalidPluginDefinitionException * @throws \Drupal\Component\Plugin\Exception\PluginNotFoundException */ - private function prepareToExportAllContent() { + private function exportAllBatch() { $content_entity_types = $this->deployManager->getContentEntityTypes(); if ($this->forceUpdate) { @@ -434,18 +549,43 @@ class Exporter { $entity_ids = array_values($query->execute()); foreach ($entity_ids as $entity_id) { - /** @var \Drupal\Core\Entity\ContentEntityInterface $entity */ - $entity = $this->entityTypeManager->getStorage($entity_type)->load($entity_id); - - if ($time && $entity instanceof EntityChangedInterface && $entity->getChangedTimeAcrossTranslations() < $time) { - continue; - } - - $exported_entity = $this->getSerializedContent($entity); - $this->addExportedEntity($exported_entity); + $operations[] = [ + [$this, 'exportBatchDefault'], + [$entity_type, $entity_id], + ]; } } } + + // Set up batch information. + $batch = [ + 'title' => t('Exporting content'), + 'operations' => $operations, + 'finished' => [$this, 'exportFinished'], + ]; + + batch_set($batch); + + if (PHP_SAPI === 'cli') { + drush_backend_batch_process(); + } + } + + /** + * Callback function to handle batch processing completion. + * + * @param bool $success + * Indicates whether the batch processing was successful. + */ + public function exportFinished($success, $results, $operations) { + if ($success) { + // Batch processing completed successfully. + \Drupal::messenger()->addMessage(t('Batch export completed successfully.')); + } + else { + // Batch processing encountered an error. + \Drupal::messenger()->addMessage(t('An error occurred during the batch export process.'), 'error'); + } } /** @@ -653,7 +793,7 @@ class Exporter { $configuration = $component->get('configuration'); if ($configuration['id'] === 'inline_block:html_block' && !empty($configuration['block_revision_id'])) { $block_revision_id = $configuration['block_revision_id']; - $block_revision = \Drupal::entityTypeManager() + $block_revision = $this->entityTypeManager ->getStorage('block_content') ->loadRevision($block_revision_id); $entity_dependencies[] = $block_revision; @@ -662,7 +802,7 @@ class Exporter { else { if (substr($configuration['id'], 0, 14) === 'block_content:') { if ($block_uuid = substr($configuration['id'], 14)) { - $block_loaded_by_uuid = \Drupal::entityTypeManager() + $block_loaded_by_uuid = $this->entityTypeManager ->getStorage('block_content') ->loadByProperties(['uuid' => $block_uuid]); $entity_dependencies[] = reset($block_loaded_by_uuid); @@ -681,6 +821,85 @@ class Exporter { return $entity_dependencies; } + /** + * Returns all processed text references of an entity. + * + * @param \Drupal\Core\Entity\ContentEntityInterface $entity + * The entity. + * + * @return \Drupal\Core\Entity\ContentEntityInterface[] + * Keyed array of entities indexed by entity type and ID. + */ + private function getEntityProcessedTextDependencies(ContentEntityInterface $entity) { + $entity_dependencies = []; + + $field_definitions = $entity->getFieldDefinitions(); + $entity_type_id = $entity->getEntityTypeId(); + $bundle = $entity->bundle(); + + $languages = $this->languageManager->getLanguages(); + $default_language = $this->languageManager->getDefaultLanguage()->getId(); + + foreach ($languages as $langcode => $language) { + $entity = $this->entityRepository->getTranslationFromContext($entity, $langcode); + + foreach ($field_definitions as $key => $field) { + $field_config = $field->getConfig($bundle); + $field_storage_definition = $field_config->getFieldStorageDefinition(); + $field_storage_property_definition = $field_storage_definition->getPropertyDefinitions(); + + // If the field is not translatable and we are working on a translation don't bother scanning. + if (!$field->isTranslatable() && $langcode != $default_language) { + continue; + } + + if (isset($field_storage_property_definition['processed'])) { + $field_name = $field_config->getName(); + $field_data = $entity->get($field_name)->getString(); + + $dom = Html::load($field_data); + $xpath = new \DOMXPath($dom); + + // Iterate over all elements with a data-entity-type attribute. + foreach ($xpath->query('//*[@data-entity-type]') as $node) { + $entity_type = $node->getAttribute('data-entity-type'); + $uuid = $node->getAttribute('data-entity-uuid'); + + // Only add the dependency if it does not already exist. + if (!in_array($uuid, $entity_dependencies)) { + $entity_loaded_by_uuid = $this->entityTypeManager + ->getStorage($entity_type) + ->loadByProperties(['uuid' => $uuid]); + $entity_dependencies[] = reset($entity_loaded_by_uuid); + } + } + } + } + } + + return $entity_dependencies; + } + + /** + * Returns an entity based on storage type and UUID. + * + * @param $storage_type + * The type of entity storage. + * @param $uuid + * The UUID of the entity to load. + * + * @return \Drupal\Core\Entity\ContentEntityInterface + * An entity. + */ + function getEntityByUuid($storage_type, $uuid) { + $entity = $this->entityTypeManager + ->getStorage($storage_type) + ->loadByProperties(['uuid' => $uuid]); + $entity = reset($entity); + + return $entity; + } + /** * Returns all referenced entities of an entity. * @@ -698,9 +917,18 @@ class Exporter { * Keyed array of entities indexed by entity type and ID. */ private function getEntityReferencesRecursive(ContentEntityInterface $entity, $depth = 0, array &$indexed_dependencies = []) { + $config = $this->config->get('default_content_deploy.content_directory'); + $text_dependencies = $config->get('text_dependencies'); + $entity_dependencies = $entity->referencedEntities(); $entity_layout_builder_dependencies = $this->getEntityLayoutBuilderDependencies($entity); - $entity_dependencies = array_merge($entity_dependencies, $entity_layout_builder_dependencies); + + $entity_processed_text_dependencies = []; + if ($text_dependencies) { + $entity_processed_text_dependencies = $this->getEntityProcessedTextDependencies($entity); + } + + $entity_dependencies = array_merge($entity_dependencies, $entity_layout_builder_dependencies, $entity_processed_text_dependencies); foreach ($entity_dependencies as $dependent_entity) { // Config entities should not be exported but rather provided by default diff --git a/src/Form/ImportForm.php b/src/Form/ImportForm.php index 6625f47..ab3c5a1 100644 --- a/src/Form/ImportForm.php +++ b/src/Form/ImportForm.php @@ -163,8 +163,6 @@ class ImportForm extends FormBase { } $this->importer->setForceOverride($force_override); - $this->importer->prepareForImport(); - $this->addResultMessage(); $this->importer->import(); } catch (\Exception $exception) { @@ -172,27 +170,4 @@ class ImportForm extends FormBase { } } - /** - * Add a message with importing results. - */ - private function addResultMessage() { - $result = $this->importer->getResult(); - $array_column = array_column($result, 'status'); - $count = array_count_values($array_column); - - $this->messenger->addMessage($this->t('Created: @count', [ - '@count' => isset($count['create']) ? $count['create'] : 0, - ])); - - $this->messenger->addMessage($this->t('Updated: @count', [ - '@count' => isset($count['update']) ? $count['update'] : 0, - ])); - - $this->messenger->addMessage($this->t('Skipped: @count', [ - '@count' => isset($count['skip']) ? $count['skip'] : 0, - ])); - - return $this; - } - } diff --git a/src/Form/SettingsForm.php b/src/Form/SettingsForm.php index e888a16..ef70119 100644 --- a/src/Form/SettingsForm.php +++ b/src/Form/SettingsForm.php @@ -9,6 +9,10 @@ namespace Drupal\default_content_deploy\Form; use Drupal\Core\Form\ConfigFormBase; use Drupal\Core\Form\FormStateInterface; +use Drupal\Core\Config\ConfigFactoryInterface; +use Drupal\Core\Entity\ContentEntityTypeInterface; +use Drupal\Core\Entity\EntityTypeManagerInterface; +use Symfony\Component\DependencyInjection\ContainerInterface; /** * Settings form. @@ -20,6 +24,31 @@ class SettingsForm extends ConfigFormBase { */ const DIRECTORY = 'default_content_deploy.content_directory'; + /** + * The Entity type manager. + * + * @var \Drupal\Core\Entity\EntityTypeManagerInterface + */ + protected $entityTypeManager; + + /** + * {@inheritdoc} + */ + public function __construct(ConfigFactoryInterface $config_factory, EntityTypeManagerInterface $entity_type_manager) { + parent::__construct($config_factory); + $this->entityTypeManager = $entity_type_manager; + } + + /** + * {@inheritdoc} + */ + public static function create(ContainerInterface $container) { + return new static( + $container->get('config.factory'), + $container->get('entity_type.manager'), + ); + } + /** * {@inheritdoc} */ @@ -49,6 +78,42 @@ class SettingsForm extends ConfigFormBase { '#description' => 'Specify the path relative to index.php. For example: ../content', ]; + $all_entity_types = $this->entityTypeManager->getDefinitions(); + $content_entity_types = []; + + // Filter the entity types. + /** @var \Drupal\Core\Entity\EntityTypeInterface[] $entity_type_options */ + foreach ($all_entity_types as $entity_type) { + if (($entity_type instanceof ContentEntityTypeInterface)) { + $content_entity_types[$entity_type->id()] = $entity_type->getLabel(); + } + } + + // Entity types. + $form['enabled_entity_types'] = [ + '#type' => 'details', + '#open' => FALSE, + '#title' => $this->t('Enabled entity types'), + '#description' => $this->t('Check which entity types should be exported by reference.'), + '#tree' => TRUE, + ]; + + $form['text_dependencies'] = [ + '#type' => 'checkbox', + '#title' => $this->t('Export processed text dependencies'), + '#default_value' => $config->get('text_dependencies'), + '#description' => 'If selected, embedded entities within processed text fields will be included in the export.', + ]; + + $form['enabled_entity_types']['entity_types'] = [ + '#type' => 'checkboxes', + '#title' => $this->t('Enabled entity types'), + '#options' => $content_entity_types, + // If no custom settings exist, content entities are enabled by default. + '#default_value' => $config->get('enabled_entity_types') ?: array_keys($content_entity_types), + '#required' => TRUE, + ]; + return parent::buildForm($form, $form_state); } @@ -58,6 +123,8 @@ class SettingsForm extends ConfigFormBase { public function submitForm(array &$form, FormStateInterface $form_state) { $this->configFactory->getEditable(static::DIRECTORY) ->set('content_directory', $form_state->getValue('content_directory')) + ->set('enabled_entity_types', array_values(array_filter($form_state->getValue('enabled_entity_types')['entity_types']))) + ->set('text_dependencies', $form_state->getValue('text_dependencies')) ->save(); parent::submitForm($form, $form_state); diff --git a/src/Importer.php b/src/Importer.php index 3ef0a69..c45447c 100644 --- a/src/Importer.php +++ b/src/Importer.php @@ -5,6 +5,7 @@ namespace Drupal\default_content_deploy; use Drupal\Component\EventDispatcher\ContainerAwareEventDispatcher; use Drupal\Core\Cache\CacheBackendInterface; use Drupal\Core\Database\Connection; +use Drupal\Core\DependencyInjection\DependencySerializationTrait; use Drupal\Core\Entity\EntityChangedInterface; use Drupal\Core\Entity\EntityRepositoryInterface; use Drupal\Core\Entity\EntityTypeManagerInterface; @@ -23,6 +24,8 @@ use Symfony\Component\Serializer\Serializer; */ class Importer { + use DependencySerializationTrait; + /** * Deploy manager. * @@ -237,20 +240,88 @@ class Importer { * @throws \Drupal\Component\Plugin\Exception\PluginNotFoundException * @throws \Exception */ - public function prepareForImport() { + public function import() { // @todo remove because of changes in core >= 9.2 $this->cache->delete('hal:links:relations'); $this->files = $this->scan($this->getFolder()); + // Process files in batches. + $operations = []; + $total = count($this->files); + $current = 1; + + // Initialize progress tracking here. + if (!isset($this->context['sandbox']['progress'])) { + $this->context['sandbox']['progress'] = 0; + $this->context['sandbox']['total'] = $total; + } + + if ($total == 0) { + \Drupal::messenger()->addMessage(t('Nothing to import.')); + return; + } + foreach ($this->files as $file) { - $uuid = str_replace('.json', '', $file->name); + $prepare_operations[] = [ + [$this, 'decodeFile'], + [$file, $current, $total], + ]; - if (!isset($this->dataToImport[$uuid])) { - $this->decodeFile($file); - } + $import_operations[] = [ + [$this, 'importEntity'], + [$total, $current], + ]; + + $current++; } - return $this; + $operations = array_merge($prepare_operations, $import_operations); + + $batch = [ + 'title' => t('Importing Default Content'), + 'operations' => $operations, + 'finished' => [$this, 'importFinished'], + ]; + + batch_set($batch); + + if (PHP_SAPI === 'cli') { + drush_backend_batch_process(); + } + } + + /** + * Callback function to handle batch pre-processing completion. + * + * @param bool $success + * Indicates whether the batch processing was successful. + */ + public function importFinished($success, $results, $operations) { + if ($success) { + // Batch processing completed successfully. + \Drupal::messenger()->addMessage(t('Batch import completed successfully.')); + + // Output result counts. + $results = !empty($results['data_to_import']) ? $results['data_to_import'] : []; + $array_column = array_column($results, 'status'); + $count = array_count_values($array_column); + + \Drupal::messenger()->addMessage(t('Created: @count', [ + '@count' => isset($count['create']) ? $count['create'] : 0, + ])); + + \Drupal::messenger()->addMessage(t('Updated: @count', [ + '@count' => isset($count['update']) ? $count['update'] : 0, + ])); + + \Drupal::messenger()->addMessage(t('Skipped: @count', [ + '@count' => isset($count['skip']) ? $count['skip'] : 0, + ])); + } + else { + // Batch processing encountered an error. + \Drupal::messenger()->addMessage(t('An error occurred during the batch import process.'), 'error'); + } } /** @@ -291,93 +362,98 @@ class Importer { } /** - * Import to entity. + * Imports a single entity as part of the batch process. * - * @throws \Drupal\Component\Plugin\Exception\InvalidPluginDefinitionException - * @throws \Drupal\Component\Plugin\Exception\PluginNotFoundException - * @throws \Drupal\Core\Entity\EntityStorageException + * @param string $uuid + * The UUID of the entity to import. + * @param array $data + * The data of the entity to import. */ - public function import() { - $files = $this->dataToImport; + public function importEntity($total, $current, &$context) { + $batch = batch_get(); + $data_to_import = $batch['sets'][0]['results']['data_to_import']; + + $keys = array_keys($data_to_import); + // We are accessing an array, so subtract 1 to start at zero. + $uuid = $keys[$current - 1]; + $data = $data_to_import[$uuid]; if (PHP_SAPI === 'cli') { $root_user = $this->entityTypeManager->getStorage('user')->load(1); $this->accountSwitcher->switchTo($root_user); } - // All entities with entity references will be imported two times to ensure - // that all entity references are present and valid. Path aliases will be - // imported last to have a chance to rewrite them to the new ids of newly - // created entities. - for ($i = 0; $i <= 2; $i++) { - foreach ($files as $uuid => &$file) { - $entity_type = $file['entity_type_id']; - if ($file['status'] !== 'skip') { - if ( - ($i !== 2 && $entity_type === 'path_alias') || - ($i === 2 && $entity_type !== 'path_alias') - ) { - continue; - } - $this->linkManager->setLinkDomain($this->getLinkDomain($file)); - $class = $this->entityTypeManager->getDefinition($entity_type)->getClass(); - $needs_second_run = $this->preDenormalize($file, $entity_type); - - /** @var \Drupal\Core\Entity\ContentEntityInterface $entity */ - $entity = $this->serializer->denormalize($file['data'], $class, 'hal_json', ['request_method' => 'POST']); - $this->eventDispatcher->dispatch(new PreSaveEntityEvent($entity, $file['data'])); - $entity->enforceIsNew($file['is_new']); - $entity->save(); - $this->entityIdLookup[$uuid] = $entity->id(); - - if ($file['is_new']) { - $this->newUuids[] = $uuid; - } + $entity_type_id = $data['entity_type_id']; + $entity_type_object = $this->entityTypeManager->getDefinition($entity_type_id); - if ($entity_type === 'user') { - // Workaround: store the hashed password directly in the database - // and avoid the entity API which doesn't provide support for - // setting password hashes directly. - $hashed_pass = $file['data']['pass'][0]['value'] ?? FALSE; - if ($hashed_pass) { - $this->database->update('users_field_data') - ->fields([ - 'pass' => $hashed_pass, - ]) - ->condition('uid', $entity->id(), '=') - ->execute(); - } - } + // Keys of entity. + $key_id = $entity_type_object->getKey('id'); + $key_revision_id = $entity_type_object->getKey('revision'); - if ((!$needs_second_run && empty($file['references'])) || $i === 1) { - // Don't handle entities without references twice. Don't handle - // entities with references again in the third run for path aliases. - unset($files[$uuid]); - } - else { - // In the second run new entities should be updated. - $file['status'] = 'update'; - $file['is_new'] = FALSE; - $file['data'][$file['key_id']][0]['value'] = $entity->id(); - } - } - elseif ($i === 0) { - // Get the entity ID of a skipped referenced item in the first run to - // enable a target ID correction in referencing entities in the second - // and third run. - $entity = $this->entityRepository->loadEntityByUuid($entity_type, $uuid); - $this->entityIdLookup[$uuid] = $entity->id(); - } + // Check if the entity already exists. + $entity = $this->entityRepository->loadEntityByUuid($entity_type_id, $uuid); + + if ($entity) { + // Entity already exists, update it. + $this->linkManager->setLinkDomain($this->getLinkDomain($data)); + $current_entity_decoded = $this->serializer->decode($this->exporter->getSerializedContent($entity), 'hal_json'); + $diff = ArrayDiffMultidimensional::looseComparison($data['data'], $current_entity_decoded); + + // Update the entity if it's different from the existing one. + if ($diff) { + $this->preAddToImport($data); + $this->addToImport($data); + $data['status'] = 'update'; + $data['is_new'] = FALSE; + } + else { + $data['status'] = 'skip'; } - unset($file); + + // @todo is this still needed? + $this->linkManager->setLinkDomain(FALSE); } + else { + // Entity doesn't exist, create it. + $this->linkManager->setLinkDomain($this->getLinkDomain($data)); + $class = $entity_type_object->getClass(); + $entity = $this->serializer->denormalize($data['data'], $class, 'hal_json', ['request_method' => 'POST']); + $entity->enforceIsNew($data['is_new']); + $entity->save(); + $this->entityIdLookup[$uuid] = $entity->id(); + + // If the entity type is 'user', update the password directly in the database. + if ($entity_type_id === 'user') { + $hashed_pass = $data['data']['pass'][0]['value'] ?? FALSE; + if ($hashed_pass) { + $this->database->update('users_field_data') + ->fields([ + 'pass' => $hashed_pass, + ]) + ->condition('uid', $entity->id(), '=') + ->execute(); + } + } - // @todo is this still needed? - $this->linkManager->setLinkDomain(FALSE); + // Record the entity lookup. + if (isset($data['data'][$key_id][0]['value'])) { + $this->entityLookup[$entity_type_id][$data['data'][$key_id][0]['value']] = $uuid; + } + + // Add the entity to import. + $this->preAddToImport($data); + $this->addToImport($data); + $data['status'] = 'create'; + } if (PHP_SAPI === 'cli') { $this->accountSwitcher->switchBack(); } + + // Provide update progress. + $context['message'] = t('Importing entity @current of @total', ['@current' => $current, '@total' => $total]); + + return $this; } /** @@ -403,7 +479,7 @@ class Importer { * @throws \Drupal\Component\Plugin\Exception\PluginNotFoundException * @throws \Exception */ - protected function decodeFile($file) { + public function decodeFile($file, $current, $total, &$context) { // Check that this file has not been decoded already. if (array_key_exists($file->name, $this->discoveredReferences)) { return $this; @@ -425,7 +501,7 @@ class Importer { $this->discoveredReferences[$file->name] = $file; if ($references) { foreach ($references as $reference) { - $this->decodeFile($reference); + $this->decodeFile($reference, $current, $total, $context); } } @@ -440,6 +516,11 @@ class Importer { $this->preAddToImport($data_to_import); $this->addToImport($data_to_import); + // Pass export data to results. + $uuid = $data_to_import['data']['uuid'][0]['value']; + $context['results']['data_to_import'][$uuid] = $data_to_import; + $context['message'] = t('Preparing entity @current of @total for import', ['@current' => $current, '@total' => $total]); + return $this; }