diff --git a/default_content_deploy.services.yml b/default_content_deploy.services.yml index 11d69f0..25121c4 100644 --- a/default_content_deploy.services.yml +++ b/default_content_deploy.services.yml @@ -1,7 +1,7 @@ services: default_content_deploy.importer: class: Drupal\default_content_deploy\Importer - arguments: ['@serializer', '@entity_type.manager', '@hal.link_manager', '@account_switcher', '@default_content_deploy.manager', '@entity.repository', '@cache.default', '@default_content_deploy.exporter', '@database', '@event_dispatcher'] + arguments: ['@serializer', '@entity_type.manager', '@hal.link_manager', '@account_switcher', '@default_content_deploy.manager', '@entity.repository', '@cache.default', '@default_content_deploy.exporter', '@database', '@event_dispatcher', '@keyvalue'] default_content_deploy.exporter: class: Drupal\default_content_deploy\Exporter arguments: ['@database', '@default_content_deploy.manager', '@entity_type.manager', '@serializer', '@account_switcher', '@file_system', '@hal.link_manager', '@event_dispatcher', '@module_handler', '@config.factory', '@language_manager', '@entity.repository'] diff --git a/src/Commands/DefaultContentDeployCommands.php b/src/Commands/DefaultContentDeployCommands.php index acaaa4b..deb819d 100644 --- a/src/Commands/DefaultContentDeployCommands.php +++ b/src/Commands/DefaultContentDeployCommands.php @@ -176,7 +176,11 @@ class DefaultContentDeployCommands extends DrushCommands { // Set text_dependencies option. $text_dependencies = $options['text_dependencies']; - $text_dependencies = filter_var($text_dependencies, FILTER_VALIDATE_BOOLEAN); + + if (!is_null($text_dependencies)) { + $text_dependencies = filter_var($text_dependencies, FILTER_VALIDATE_BOOLEAN); + } + $this->exporter->setTextDependencies($text_dependencies); $this->exporter->setMode('reference'); @@ -297,12 +301,8 @@ class DefaultContentDeployCommands extends DrushCommands { $this->importer->setFolder($options['folder']); } - $this->importer->prepareForImport(); - $this->displayImportResult(); - - if (!$this->isAllSkip() && $this->io()->confirm(dt('Do you really want to continue?'))) { + if ($this->io()->confirm(dt('Do you really want to continue?'))) { $this->importer->import(); - $this->io()->success(dt('Content has been imported.')); } } @@ -357,37 +357,6 @@ class DefaultContentDeployCommands extends DrushCommands { } } - /** - * Display info before/after import. - */ - private function displayImportResult() { - $result = $this->importer->getResult(); - $array_column = array_column($result, 'status'); - $count = array_count_values($array_column); - - if ($this->output()->isVerbose()) { - $table = new Table($this->output()); - $table->setHeaders(['Action', 'Entity Type', 'UUID']); - foreach ($result as $uuid => $data) { - $table->addRow([$data['status'], $data['entity_type_id'], $uuid]); - } - $table->render(); - $this->output()->writeln(dt('Summary:')); - } - - $this->output()->writeln(dt('- created: @count', [ - '@count' => isset($count['create']) ? $count['create'] : 0, - ])); - - $this->output()->writeln(dt('- updated: @count', [ - '@count' => isset($count['update']) ? $count['update'] : 0, - ])); - - $this->output()->writeln(dt('- skipped: @count', [ - '@count' => isset($count['skip']) ? $count['skip'] : 0, - ])); - } - /** * Is update or create not exist. * diff --git a/src/Exporter.php b/src/Exporter.php index 20d6557..2f3ae34 100644 --- a/src/Exporter.php +++ b/src/Exporter.php @@ -6,6 +6,7 @@ use Drupal\Component\EventDispatcher\ContainerAwareEventDispatcher; use Drupal\Component\Utility\Html; use Drupal\Core\Config\ConfigFactoryInterface; use Drupal\Core\Database\Connection; +use Drupal\Core\DependencyInjection\DependencySerializationTrait; use Drupal\Core\Entity\ContentEntityInterface; use Drupal\Core\Entity\EntityChangedInterface; use Drupal\Core\Entity\EntityRepositoryInterface; @@ -26,6 +27,8 @@ use Drupal\layout_builder\SectionComponent; */ class Exporter { + use DependencySerializationTrait; + /** * The config factory. * @@ -419,32 +422,31 @@ class Exporter { public function export() { switch ($this->mode) { case 'default': - $this->prepareToExport(); + $this->exportBatch(); break; case 'reference': - $this->prepareToExportWithReference(); + $this->exportBatch(TRUE); break; case 'all': - $this->prepareToExportAllContent(); + $this->exportAllBatch(); break; } - // Edit and export all entities to folder. - $this->editEntityData(); - $this->writeConfigsToFolder(); - return $this; } /** - * Prepare content to export. + * Export content in batch. + * + * @param boolean $with_references + * Indicates if export should consider referenced entities. * * @throws \Drupal\Component\Plugin\Exception\InvalidPluginDefinitionException * @throws \Drupal\Component\Plugin\Exception\PluginNotFoundException */ - private function prepareToExport() { + private function exportBatch($with_references = FALSE) { $entity_type = $this->entityTypeId; $exported_entity_ids = $this->getEntityIdsForExport(); @@ -452,33 +454,112 @@ class Exporter { $this->fileSystem->deleteRecursive($this->getFolder()); } + $export_type = $with_references ? "exportBatchWithReferences" : "exportBatchDefault"; + foreach ($exported_entity_ids as $entity_id) { - /** @var \Drupal\Core\Entity\ContentEntityInterface $entity */ - $entity = $this->entityTypeManager->getStorage($entity_type)->load($entity_id); - $exported_entity = $this->getSerializedContent($entity); - $this->addExportedEntity($exported_entity); + $operations[] = [ + [$this, $export_type], + [$entity_type, $entity_id], + ]; } + + // Set up batch information. + $batch = [ + 'title' => t('Exporting content'), + 'operations' => $operations, + 'finished' => [$this, 'exportFinished'], + ]; + + batch_set($batch); + + if (PHP_SAPI === 'cli') { + drush_backend_batch_process(); + } } /** - * Prepare content with reference to export. + * Prepares and exports a single entity to a JSON file. * - * @throws \Drupal\Component\Plugin\Exception\InvalidPluginDefinitionException - * @throws \Drupal\Component\Plugin\Exception\PluginNotFoundException - */ - private function prepareToExportWithReference() { - $entity_type = $this->entityTypeId; - $exported_entity_ids = $this->getEntityIdsForExport(); + * @param string $entity_type + * The type of the entity being exported. + * @param int $entity_id + * The ID of the entity being exported. + * @param array $context + * The batch context. + */ + public function exportBatchDefault($entity_type, $entity_id, &$context) { + // Prepare export of entity. + $entity = $this->entityTypeManager->getStorage($entity_type)->load($entity_id); + $exported_entity = $this->getSerializedContent($entity); + + // Remove or add a new fields to serialize entities data. + $entity_array = $this->serializer->decode($exported_entity, 'hal_json'); + $entity_type_object = $this->entityTypeManager->getDefinition($entity_type); + $id_key = $entity_type_object->getKey('id'); + $entity_id = $entity_array[$id_key][0]['value']; + + unset($entity_array[$entity_type_object->getKey('revision')]); + + if ($entity_type === 'user') { + $entity_array['pass'][0]['value'] = $entity->getPassword(); + } - if ($this->forceUpdate) { - $this->fileSystem->deleteRecursive($this->getFolder()); + $data = $this->serializer->serialize($entity_array, 'hal_json', [ + 'json_encode_options' => JSON_PRETTY_PRINT + ]); + + // Write serialized entity to JSON file. + $uuid = $entity->get('uuid')->value; + $entity_type_folder = "{$this->getFolder()}/{$entity_type}"; + $this->fileSystem->prepareDirectory($entity_type_folder, FileSystemInterface::CREATE_DIRECTORY); + + file_put_contents("{$entity_type_folder}/{$uuid}.json", $data); + } + + /** + * Prepares and exports a single entity to a JSON file with references. + * + * @param string $entity_type + * The type of the entity being exported. + * @param int $entity_id + * The ID of the entity being exported. + * @param array $context + * The batch context. + */ + public function exportBatchWithReferences($entity_type, $entity_id, &$context) { + // Get referenced entities. + $entity = $this->entityTypeManager->getStorage($entity_type)->load($entity_id); + + $entities = []; + if ($entity instanceof ContentEntityInterface) { + $indexed_dependencies = [$entity->uuid() => $entity]; + $entities = $this->getEntityReferencesRecursive($entity, 0, $indexed_dependencies); } - foreach ($exported_entity_ids as $entity_id) { - /** @var \Drupal\Core\Entity\ContentEntityInterface $entity */ + foreach ($entities as $uuid => $exported_entity) { + $entity_type = $exported_entity->getEntityTypeId(); + $serialized_entity = $this->getSerializedContent($exported_entity); + $entity_array = $this->serializer->decode($serialized_entity, 'hal_json'); + $entity_type_object = $this->entityTypeManager->getDefinition($entity_type); + $id_key = $entity_type_object->getKey('id'); + $entity_id = $entity_array[$id_key][0]['value']; $entity = $this->entityTypeManager->getStorage($entity_type)->load($entity_id); - $exported_entities = $this->getSerializedContentWithReferences($entity); - $this->addExportedEntity($exported_entities); + + unset($entity_array[$entity_type_object->getKey('revision')]); + + if ($entity_type === 'user') { + $entity_array['pass'][0]['value'] = $entity->getPassword(); + } + + $data = $this->serializer->serialize($entity_array, 'hal_json', [ + 'json_encode_options' => JSON_PRETTY_PRINT + ]); + + // Write serialized entity to JSON file. + $entity_type_folder = "{$this->getFolder()}/{$entity_type}"; + $this->fileSystem->prepareDirectory($entity_type_folder, FileSystemInterface::CREATE_DIRECTORY); + + file_put_contents("{$entity_type_folder}/{$uuid}.json", $data); } } @@ -488,7 +569,7 @@ class Exporter { * @throws \Drupal\Component\Plugin\Exception\InvalidPluginDefinitionException * @throws \Drupal\Component\Plugin\Exception\PluginNotFoundException */ - private function prepareToExportAllContent() { + private function exportAllBatch() { $content_entity_types = $this->deployManager->getContentEntityTypes(); if ($this->forceUpdate) { @@ -506,18 +587,43 @@ class Exporter { $entity_ids = array_values($query->execute()); foreach ($entity_ids as $entity_id) { - /** @var \Drupal\Core\Entity\ContentEntityInterface $entity */ - $entity = $this->entityTypeManager->getStorage($entity_type)->load($entity_id); - - if ($time && $entity instanceof EntityChangedInterface && $entity->getChangedTimeAcrossTranslations() < $time) { - continue; - } - - $exported_entity = $this->getSerializedContent($entity); - $this->addExportedEntity($exported_entity); + $operations[] = [ + [$this, 'exportBatchDefault'], + [$entity_type, $entity_id], + ]; } } } + + // Set up batch information. + $batch = [ + 'title' => t('Exporting content'), + 'operations' => $operations, + 'finished' => [$this, 'exportFinished'], + ]; + + batch_set($batch); + + if (PHP_SAPI === 'cli') { + drush_backend_batch_process(); + } + } + + /** + * Callback function to handle batch processing completion. + * + * @param bool $success + * Indicates whether the batch processing was successful. + */ + public function exportFinished($success, $results, $operations) { + if ($success) { + // Batch processing completed successfully. + \Drupal::messenger()->addMessage(t('Batch export completed successfully.')); + } + else { + // Batch processing encountered an error. + \Drupal::messenger()->addMessage(t('An error occurred during the batch export process.'), 'error'); + } } /** @@ -672,29 +778,6 @@ class Exporter { return $content; } - /** - * Exports a single entity and all its referenced entity. - * - * @param \Drupal\Core\Entity\ContentEntityInterface $entity - * - * @return array - * - * @throws \Drupal\Component\Plugin\Exception\InvalidPluginDefinitionException - * @throws \Drupal\Component\Plugin\Exception\PluginNotFoundException - */ - private function getSerializedContentWithReferences(ContentEntityInterface $entity) { - $indexed_dependencies = [$entity->uuid() => $entity]; - $entities = $this->getEntityReferencesRecursive($entity, 0, $indexed_dependencies); - $serialized_entities = []; - - // Serialize all entities and key them by entity TYPE and uuid. - foreach ($entities as $referenced_entity) { - $serialized_entities[$referenced_entity->getEntityTypeId()][$referenced_entity->uuid()] = $this->getSerializedContent($referenced_entity); - } - - return $serialized_entities; - } - /** * Returns all layout builder referenced blocks of an entity. * @@ -763,6 +846,8 @@ class Exporter { * Keyed array of entities indexed by entity type and ID. */ private function getEntityProcessedTextDependencies(ContentEntityInterface $entity) { + $config = $this->config->get('default_content_deploy.content_directory'); + $enabled_entity_types = $config->get('enabled_entity_types'); $entity_dependencies = []; $field_definitions = $entity->getFieldDefinitions(); @@ -789,14 +874,17 @@ class Exporter { // Iterate over all elements with a data-entity-type attribute. foreach ($xpath->query('//*[@data-entity-type]') as $node) { $entity_type = $node->getAttribute('data-entity-type'); - $uuid = $node->getAttribute('data-entity-uuid'); - - // Only add the dependency if it does not already exist. - if (!in_array($uuid, $entity_dependencies)) { - $entity_loaded_by_uuid = $this->entityTypeManager - ->getStorage($entity_type) - ->loadByProperties(['uuid' => $uuid]); - $entity_dependencies[] = reset($entity_loaded_by_uuid); + + if (in_array($entity_type, $enabled_entity_types)) { + $uuid = $node->getAttribute('data-entity-uuid'); + + // Only add the dependency if it does not already exist. + if (!in_array($uuid, $entity_dependencies)) { + $entity_loaded_by_uuid = $this->entityTypeManager + ->getStorage($entity_type) + ->loadByProperties(['uuid' => $uuid]); + $entity_dependencies[] = reset($entity_loaded_by_uuid); + } } } } @@ -843,7 +931,14 @@ class Exporter { * Keyed array of entities indexed by entity type and ID. */ private function getEntityReferencesRecursive(ContentEntityInterface $entity, $depth = 0, array &$indexed_dependencies = []) { - $entity_dependencies = $entity->referencedEntities(); + $entity_dependencies = []; + $languages = $entity->getTranslationLanguages(); + + foreach (array_keys($languages) as $langcode) { + $entity = $this->entityRepository->getTranslationFromContext($entity, $langcode); + $entity_dependencies = array_merge($entity_dependencies, $entity->referencedEntities()); + } + $entity_layout_builder_dependencies = $this->getEntityLayoutBuilderDependencies($entity); $entity_processed_text_dependencies = []; @@ -862,6 +957,13 @@ class Exporter { continue; } + // Return if entity is not in the configured referencable entity types to export. + $config = $this->config->get('default_content_deploy.content_directory'); + $enabled_entity_types = $config->get('enabled_entity_types'); + if (!in_array($dependent_entity->getEntityTypeId(), $enabled_entity_types)) { + continue; + } + // Using UUID to keep dependencies unique to prevent recursion. $key = $dependent_entity->uuid(); if (isset($indexed_dependencies[$key])) { diff --git a/src/Form/ImportForm.php b/src/Form/ImportForm.php index 6625f47..ab3c5a1 100644 --- a/src/Form/ImportForm.php +++ b/src/Form/ImportForm.php @@ -163,8 +163,6 @@ class ImportForm extends FormBase { } $this->importer->setForceOverride($force_override); - $this->importer->prepareForImport(); - $this->addResultMessage(); $this->importer->import(); } catch (\Exception $exception) { @@ -172,27 +170,4 @@ class ImportForm extends FormBase { } } - /** - * Add a message with importing results. - */ - private function addResultMessage() { - $result = $this->importer->getResult(); - $array_column = array_column($result, 'status'); - $count = array_count_values($array_column); - - $this->messenger->addMessage($this->t('Created: @count', [ - '@count' => isset($count['create']) ? $count['create'] : 0, - ])); - - $this->messenger->addMessage($this->t('Updated: @count', [ - '@count' => isset($count['update']) ? $count['update'] : 0, - ])); - - $this->messenger->addMessage($this->t('Skipped: @count', [ - '@count' => isset($count['skip']) ? $count['skip'] : 0, - ])); - - return $this; - } - } diff --git a/src/Form/SettingsForm.php b/src/Form/SettingsForm.php index fa6a93e..b00ba2f 100644 --- a/src/Form/SettingsForm.php +++ b/src/Form/SettingsForm.php @@ -9,6 +9,10 @@ namespace Drupal\default_content_deploy\Form; use Drupal\Core\Form\ConfigFormBase; use Drupal\Core\Form\FormStateInterface; +use Drupal\Core\Config\ConfigFactoryInterface; +use Drupal\Core\Entity\ContentEntityTypeInterface; +use Drupal\Core\Entity\EntityTypeManagerInterface; +use Symfony\Component\DependencyInjection\ContainerInterface; /** * Settings form. @@ -20,6 +24,31 @@ class SettingsForm extends ConfigFormBase { */ const DIRECTORY = 'default_content_deploy.content_directory'; + /** + * The Entity type manager. + * + * @var \Drupal\Core\Entity\EntityTypeManagerInterface + */ + protected $entityTypeManager; + + /** + * {@inheritdoc} + */ + public function __construct(ConfigFactoryInterface $config_factory, EntityTypeManagerInterface $entity_type_manager) { + parent::__construct($config_factory); + $this->entityTypeManager = $entity_type_manager; + } + + /** + * {@inheritdoc} + */ + public static function create(ContainerInterface $container) { + return new static( + $container->get('config.factory'), + $container->get('entity_type.manager'), + ); + } + /** * {@inheritdoc} */ @@ -56,6 +85,35 @@ class SettingsForm extends ConfigFormBase { '#description' => 'If selected, embedded entities within processed text fields will be included in the export.', ]; + $all_entity_types = $this->entityTypeManager->getDefinitions(); + $content_entity_types = []; + + // Filter the entity types. + /** @var \Drupal\Core\Entity\EntityTypeInterface[] $entity_type_options */ + foreach ($all_entity_types as $entity_type) { + if (($entity_type instanceof ContentEntityTypeInterface)) { + $content_entity_types[$entity_type->id()] = $entity_type->getLabel(); + } + } + + // Entity types. + $form['enabled_entity_types'] = [ + '#type' => 'details', + '#open' => FALSE, + '#title' => $this->t('Enabled entity types'), + '#description' => $this->t('Check which entity types should be exported by reference.'), + '#tree' => TRUE, + ]; + + $form['enabled_entity_types']['entity_types'] = [ + '#type' => 'checkboxes', + '#title' => $this->t('Enabled entity types'), + '#options' => $content_entity_types, + // If no custom settings exist, content entities are enabled by default. + '#default_value' => $config->get('enabled_entity_types') ?: array_keys($content_entity_types), + '#required' => TRUE, + ]; + return parent::buildForm($form, $form_state); } @@ -65,6 +123,7 @@ class SettingsForm extends ConfigFormBase { public function submitForm(array &$form, FormStateInterface $form_state) { $this->configFactory->getEditable(static::DIRECTORY) ->set('content_directory', $form_state->getValue('content_directory')) + ->set('enabled_entity_types', array_values(array_filter($form_state->getValue('enabled_entity_types')['entity_types']))) ->set('text_dependencies', $form_state->getValue('text_dependencies')) ->save(); diff --git a/src/Importer.php b/src/Importer.php index 3ef0a69..65cb0c8 100644 --- a/src/Importer.php +++ b/src/Importer.php @@ -5,9 +5,11 @@ namespace Drupal\default_content_deploy; use Drupal\Component\EventDispatcher\ContainerAwareEventDispatcher; use Drupal\Core\Cache\CacheBackendInterface; use Drupal\Core\Database\Connection; +use Drupal\Core\DependencyInjection\DependencySerializationTrait; use Drupal\Core\Entity\EntityChangedInterface; use Drupal\Core\Entity\EntityRepositoryInterface; use Drupal\Core\Entity\EntityTypeManagerInterface; +use Drupal\Core\KeyValueStore\KeyValueFactory; use Drupal\Core\Session\AccountSwitcherInterface; use Drupal\default_content_deploy\Event\PreSaveEntityEvent; use Drupal\hal\LinkManager\LinkManagerInterface; @@ -23,6 +25,8 @@ use Symfony\Component\Serializer\Serializer; */ class Importer { + use DependencySerializationTrait; + /** * Deploy manager. * @@ -31,11 +35,11 @@ class Importer { protected $deployManager; /** - * Scanned files. + * The key-value store service. * - * @var object[] + * @var \Drupal\Core\KeyValueStore\KeyValueFactory */ - private $files; + protected $keyValueStorage; /** * Directory to import. @@ -141,6 +145,13 @@ class Importer { */ protected $newUuids = []; + /** + * The batch context. + * + * @var array + */ + protected $context; + /** * Constructs the default content deploy manager. * @@ -162,8 +173,10 @@ class Importer { * Database connection. * @param \Drupal\Component\EventDispatcher\ContainerAwareEventDispatcher $event_dispatcher * The event dispatcher. + * @param \Drupal\Core\KeyValueStore\KeyValueFactory $key_value_factory + * The key value factory. */ - public function __construct(Serializer $serializer, EntityTypeManagerInterface $entity_type_manager, LinkManagerInterface $link_manager, AccountSwitcherInterface $account_switcher, DeployManager $deploy_manager, EntityRepositoryInterface $entity_repository, CacheBackendInterface $cache, Exporter $exporter, Connection $database, ContainerAwareEventDispatcher $event_dispatcher) { + public function __construct(Serializer $serializer, EntityTypeManagerInterface $entity_type_manager, LinkManagerInterface $link_manager, AccountSwitcherInterface $account_switcher, DeployManager $deploy_manager, EntityRepositoryInterface $entity_repository, CacheBackendInterface $cache, Exporter $exporter, Connection $database, ContainerAwareEventDispatcher $event_dispatcher, KeyValueFactory $key_value_factory) { $this->serializer = $serializer; $this->entityTypeManager = $entity_type_manager; $this->linkManager = $link_manager; @@ -174,6 +187,7 @@ class Importer { $this->exporter = $exporter; $this->database = $database; $this->eventDispatcher = $event_dispatcher; + $this->keyValueStorage = $key_value_factory; } /** @@ -237,20 +251,86 @@ class Importer { * @throws \Drupal\Component\Plugin\Exception\PluginNotFoundException * @throws \Exception */ - public function prepareForImport() { - // @todo remove because of changes in core >= 9.2 - $this->cache->delete('hal:links:relations'); - $this->files = $this->scan($this->getFolder()); + public function import() { + // Get a list of all files to import. + $files = $this->scan($this->getFolder()); - foreach ($this->files as $file) { - $uuid = str_replace('.json', '', $file->name); + // Save files to KeyValueStoreInterface. + $this->keyValueStorage->get('dcd_batch_files')->set('dcd_batch_files', $files); - if (!isset($this->dataToImport[$uuid])) { - $this->decodeFile($file); - } + // Process files in batches. + $operations = []; + $total = count($files); + $current = 1; + + // Initialize progress tracking here. + if (!isset($this->context['sandbox']['progress'])) { + $this->context['sandbox']['progress'] = 0; + $this->context['sandbox']['total'] = $total; } - return $this; + if ($total == 0) { + \Drupal::messenger()->addMessage(t('Nothing to import.')); + return; + } + + foreach ($files as $file) { + $operations[] = [ + [$this, 'processFile'], + [$file, $current, $total], + ]; + + $current++; + } + + $batch = [ + 'title' => t('Importing Default Content'), + 'operations' => $operations, + 'finished' => [$this, 'importFinished'], + ]; + + batch_set($batch); + + if (PHP_SAPI === 'cli') { + drush_backend_batch_process(); + } + } + + /** + * Callback function to handle batch pre-processing completion. + * + * @param bool $success + * Indicates whether the batch processing was successful. + */ + public function importFinished($success, $results, $operations) { + // Delete the keyValue storage data. + $this->keyValueStorage->get('dcd_batch_files')->delete('dcd_batch_files'); + + if ($success) { + // Batch processing completed successfully. + \Drupal::messenger()->addMessage(t('Batch import completed successfully.')); + + // Output result counts. + $results = !empty($results['data_to_import']) ? $results['data_to_import'] : []; + $array_column = array_column($results, 'status'); + $count = array_count_values($array_column); + + \Drupal::messenger()->addMessage(t('Created: @count', [ + '@count' => isset($count['create']) ? $count['create'] : 0, + ])); + + \Drupal::messenger()->addMessage(t('Updated: @count', [ + '@count' => isset($count['update']) ? $count['update'] : 0, + ])); + + \Drupal::messenger()->addMessage(t('Skipped: @count', [ + '@count' => isset($count['skip']) ? $count['skip'] : 0, + ])); + } + else { + // Batch processing encountered an error. + \Drupal::messenger()->addMessage(t('An error occurred during the batch import process.'), 'error'); + } } /** @@ -290,120 +370,20 @@ class Importer { return $files; } - /** - * Import to entity. - * - * @throws \Drupal\Component\Plugin\Exception\InvalidPluginDefinitionException - * @throws \Drupal\Component\Plugin\Exception\PluginNotFoundException - * @throws \Drupal\Core\Entity\EntityStorageException - */ - public function import() { - $files = $this->dataToImport; - - if (PHP_SAPI === 'cli') { - $root_user = $this->entityTypeManager->getStorage('user')->load(1); - $this->accountSwitcher->switchTo($root_user); - } - - // All entities with entity references will be imported two times to ensure - // that all entity references are present and valid. Path aliases will be - // imported last to have a chance to rewrite them to the new ids of newly - // created entities. - for ($i = 0; $i <= 2; $i++) { - foreach ($files as $uuid => &$file) { - $entity_type = $file['entity_type_id']; - if ($file['status'] !== 'skip') { - if ( - ($i !== 2 && $entity_type === 'path_alias') || - ($i === 2 && $entity_type !== 'path_alias') - ) { - continue; - } - $this->linkManager->setLinkDomain($this->getLinkDomain($file)); - $class = $this->entityTypeManager->getDefinition($entity_type)->getClass(); - $needs_second_run = $this->preDenormalize($file, $entity_type); - - /** @var \Drupal\Core\Entity\ContentEntityInterface $entity */ - $entity = $this->serializer->denormalize($file['data'], $class, 'hal_json', ['request_method' => 'POST']); - $this->eventDispatcher->dispatch(new PreSaveEntityEvent($entity, $file['data'])); - $entity->enforceIsNew($file['is_new']); - $entity->save(); - $this->entityIdLookup[$uuid] = $entity->id(); - - if ($file['is_new']) { - $this->newUuids[] = $uuid; - } - - if ($entity_type === 'user') { - // Workaround: store the hashed password directly in the database - // and avoid the entity API which doesn't provide support for - // setting password hashes directly. - $hashed_pass = $file['data']['pass'][0]['value'] ?? FALSE; - if ($hashed_pass) { - $this->database->update('users_field_data') - ->fields([ - 'pass' => $hashed_pass, - ]) - ->condition('uid', $entity->id(), '=') - ->execute(); - } - } - - if ((!$needs_second_run && empty($file['references'])) || $i === 1) { - // Don't handle entities without references twice. Don't handle - // entities with references again in the third run for path aliases. - unset($files[$uuid]); - } - else { - // In the second run new entities should be updated. - $file['status'] = 'update'; - $file['is_new'] = FALSE; - $file['data'][$file['key_id']][0]['value'] = $entity->id(); - } - } - elseif ($i === 0) { - // Get the entity ID of a skipped referenced item in the first run to - // enable a target ID correction in referencing entities in the second - // and third run. - $entity = $this->entityRepository->loadEntityByUuid($entity_type, $uuid); - $this->entityIdLookup[$uuid] = $entity->id(); - } - } - unset($file); - } - - // @todo is this still needed? - $this->linkManager->setLinkDomain(FALSE); - - if (PHP_SAPI === 'cli') { - $this->accountSwitcher->switchBack(); - } - } - - /** - * Gets url from file for set to Link manager. - * - * @param array $file - */ - protected function getLinkDomain($file) { - $link = $file['data']['_links']['type']['href']; - $url_data = parse_url($link); - $host = "{$url_data['scheme']}://{$url_data['host']}"; - return (!isset($url_data['port'])) ? $host : "{$host}:{$url_data['port']}"; - } - /** * Prepare file to import. * - * @param $file + * @param $file_name + * @param $file_path + * @param $current + * @param $total + * @param $context * * @return $this * - * @throws \Drupal\Component\Plugin\Exception\InvalidPluginDefinitionException - * @throws \Drupal\Component\Plugin\Exception\PluginNotFoundException * @throws \Exception */ - protected function decodeFile($file) { + public function processFile($file, $current, $total, &$context) { // Check that this file has not been decoded already. if (array_key_exists($file->name, $this->discoveredReferences)) { return $this; @@ -425,7 +405,7 @@ class Importer { $this->discoveredReferences[$file->name] = $file; if ($references) { foreach ($references as $reference) { - $this->decodeFile($reference); + $this->processFile($reference, $current, $total, $context); } } @@ -438,11 +418,119 @@ class Importer { ]; $this->preAddToImport($data_to_import); - $this->addToImport($data_to_import); + + // Import entity. + $batch = batch_get(); + $data_to_import[$file->name] = $data_to_import; + $this->importEntity($data_to_import, $total, $current, $context); + + // Pass export data to results. + $uuid = $data_to_import['data']['uuid'][0]['value']; + $context['results']['data_to_import'][$uuid]['status'] = $data_to_import['status']; + $context['message'] = t('Processing entity @current of @total for import', ['@current' => $current, '@total' => $total]); + + return $this; + } + + /** + * Imports a single entity as part of the batch process. + * + * @param string $uuid + * The UUID of the entity to import. + * @param array $data + * The data of the entity to import. + */ + public function importEntity($data, $total, $current, &$context) { + $uuid = $data['data']['uuid'][0]['value']; + + if (PHP_SAPI === 'cli') { + $root_user = $this->entityTypeManager->getStorage('user')->load(1); + $this->accountSwitcher->switchTo($root_user); + } + + $entity_type_id = $data['entity_type_id']; + $entity_type_object = $this->entityTypeManager->getDefinition($entity_type_id); + + // Keys of entity. + $key_id = $entity_type_object->getKey('id'); + $key_revision_id = $entity_type_object->getKey('revision'); + + // Check if the entity already exists. + $entity = $this->entityRepository->loadEntityByUuid($entity_type_id, $uuid); + + if ($entity) { + // Entity already exists, update it. + $this->linkManager->setLinkDomain($this->getLinkDomain($data)); + $current_entity_decoded = $this->serializer->decode($this->exporter->getSerializedContent($entity), 'hal_json'); + $diff = ArrayDiffMultidimensional::looseComparison($data['data'], $current_entity_decoded); + + // Update the entity if it's different from the existing one. + if ($diff) { + $this->preAddToImport($data); + $data['status'] = 'update'; + $data['is_new'] = FALSE; + } + else { + $data['status'] = 'skip'; + } + + // @todo is this still needed? + $this->linkManager->setLinkDomain(FALSE); + } + else { + // Entity doesn't exist, create it. + $this->linkManager->setLinkDomain($this->getLinkDomain($data)); + $class = $entity_type_object->getClass(); + $entity = $this->serializer->denormalize($data['data'], $class, 'hal_json', ['request_method' => 'POST']); + $entity->enforceIsNew($data['is_new']); + $entity->save(); + $this->entityIdLookup[$uuid] = $entity->id(); + + // If the entity type is 'user', update the password directly in the database. + if ($entity_type_id === 'user') { + $hashed_pass = $data['data']['pass'][0]['value'] ?? FALSE; + if ($hashed_pass) { + $this->database->update('users_field_data') + ->fields([ + 'pass' => $hashed_pass, + ]) + ->condition('uid', $entity->id(), '=') + ->execute(); + } + } + + // Record the entity lookup. + if (isset($data['data'][$key_id][0]['value'])) { + $this->entityLookup[$entity_type_id][$data['data'][$key_id][0]['value']] = $uuid; + } + + // Add the entity to import. + $this->preAddToImport($data); + $data['status'] = 'create'; + } + + if (PHP_SAPI === 'cli') { + $this->accountSwitcher->switchBack(); + } + + // Provide update progress. + $context['message'] = t('Importing entity @current of @total', ['@current' => $current, '@total' => $total]); return $this; } + /** + * Gets url from file for set to Link manager. + * + * @param array $file + */ + protected function getLinkDomain($file) { + $link = $file['data']['_links']['type']['href']; + $url_data = parse_url($link); + $host = "{$url_data['scheme']}://{$url_data['host']}"; + return (!isset($url_data['port'])) ? $host : "{$host}:{$url_data['port']}"; + } + /** * Here we can edit data`s value before importing. * @@ -553,20 +641,6 @@ class Importer { return $needs_second_run; } - /** - * Adding prepared data for import. - * - * @param $data - * - * @return $this - */ - protected function addToImport($data) { - $uuid = $data['data']['uuid'][0]['value']; - $this->dataToImport[$uuid] = $data; - - return $this; - } - /** * Get all reference by entity array content. * @@ -575,6 +649,8 @@ class Importer { * @return array */ private function getReferences(array $content) { + $files = $this->getFiles(); + $references = []; if (isset($content['_embedded'])) { @@ -585,7 +661,7 @@ class Importer { $path = $this->getPathToFileByName($uuid); if ($path) { - $references[$uuid] = $this->files[$path]; + $references[$uuid] = $files[$path]; } } } @@ -603,10 +679,23 @@ class Importer { * @return false|int|string */ private function getPathToFileByName($name) { - $array_column = array_column($this->files, 'name', 'uri'); + $files = $this->getFiles(); + $array_column = array_column($files, 'name', 'uri'); return array_search($name . '.json', $array_column); } + /** + * Gets all JSON files for import. + * + * @return array|NULL + */ + private function getFiles() { + // Get files from keyValueStorage. + $files = $this->keyValueStorage->get('dcd_batch_files')->get('dcd_batch_files'); + + return $files; + } + /** * Get Entity type ID by link. *