diff --git a/default_content_deploy.services.yml b/default_content_deploy.services.yml index 11d69f0..e6fd265 100644 --- a/default_content_deploy.services.yml +++ b/default_content_deploy.services.yml @@ -1,7 +1,7 @@ services: default_content_deploy.importer: class: Drupal\default_content_deploy\Importer - arguments: ['@serializer', '@entity_type.manager', '@hal.link_manager', '@account_switcher', '@default_content_deploy.manager', '@entity.repository', '@cache.default', '@default_content_deploy.exporter', '@database', '@event_dispatcher'] + arguments: ['@serializer', '@entity_type.manager', '@hal.link_manager', '@account_switcher', '@default_content_deploy.manager', '@entity.repository', '@cache.default', '@default_content_deploy.exporter', '@database', '@event_dispatcher', '@keyvalue', '@config.factory'] default_content_deploy.exporter: class: Drupal\default_content_deploy\Exporter arguments: ['@database', '@default_content_deploy.manager', '@entity_type.manager', '@serializer', '@account_switcher', '@file_system', '@hal.link_manager', '@event_dispatcher', '@module_handler', '@config.factory', '@language_manager', '@entity.repository'] diff --git a/src/Commands/DefaultContentDeployCommands.php b/src/Commands/DefaultContentDeployCommands.php index acaaa4b..deb819d 100644 --- a/src/Commands/DefaultContentDeployCommands.php +++ b/src/Commands/DefaultContentDeployCommands.php @@ -176,7 +176,11 @@ class DefaultContentDeployCommands extends DrushCommands { // Set text_dependencies option. $text_dependencies = $options['text_dependencies']; - $text_dependencies = filter_var($text_dependencies, FILTER_VALIDATE_BOOLEAN); + + if (!is_null($text_dependencies)) { + $text_dependencies = filter_var($text_dependencies, FILTER_VALIDATE_BOOLEAN); + } + $this->exporter->setTextDependencies($text_dependencies); $this->exporter->setMode('reference'); @@ -297,12 +301,8 @@ class DefaultContentDeployCommands extends DrushCommands { $this->importer->setFolder($options['folder']); } - $this->importer->prepareForImport(); - $this->displayImportResult(); - - if (!$this->isAllSkip() && $this->io()->confirm(dt('Do you really want to continue?'))) { + if ($this->io()->confirm(dt('Do you really want to continue?'))) { $this->importer->import(); - $this->io()->success(dt('Content has been imported.')); } } @@ -357,37 +357,6 @@ class DefaultContentDeployCommands extends DrushCommands { } } - /** - * Display info before/after import. - */ - private function displayImportResult() { - $result = $this->importer->getResult(); - $array_column = array_column($result, 'status'); - $count = array_count_values($array_column); - - if ($this->output()->isVerbose()) { - $table = new Table($this->output()); - $table->setHeaders(['Action', 'Entity Type', 'UUID']); - foreach ($result as $uuid => $data) { - $table->addRow([$data['status'], $data['entity_type_id'], $uuid]); - } - $table->render(); - $this->output()->writeln(dt('Summary:')); - } - - $this->output()->writeln(dt('- created: @count', [ - '@count' => isset($count['create']) ? $count['create'] : 0, - ])); - - $this->output()->writeln(dt('- updated: @count', [ - '@count' => isset($count['update']) ? $count['update'] : 0, - ])); - - $this->output()->writeln(dt('- skipped: @count', [ - '@count' => isset($count['skip']) ? $count['skip'] : 0, - ])); - } - /** * Is update or create not exist. * diff --git a/src/Exporter.php b/src/Exporter.php index 20d6557..2f3ae34 100644 --- a/src/Exporter.php +++ b/src/Exporter.php @@ -6,6 +6,7 @@ use Drupal\Component\EventDispatcher\ContainerAwareEventDispatcher; use Drupal\Component\Utility\Html; use Drupal\Core\Config\ConfigFactoryInterface; use Drupal\Core\Database\Connection; +use Drupal\Core\DependencyInjection\DependencySerializationTrait; use Drupal\Core\Entity\ContentEntityInterface; use Drupal\Core\Entity\EntityChangedInterface; use Drupal\Core\Entity\EntityRepositoryInterface; @@ -26,6 +27,8 @@ use Drupal\layout_builder\SectionComponent; */ class Exporter { + use DependencySerializationTrait; + /** * The config factory. * @@ -419,32 +422,31 @@ class Exporter { public function export() { switch ($this->mode) { case 'default': - $this->prepareToExport(); + $this->exportBatch(); break; case 'reference': - $this->prepareToExportWithReference(); + $this->exportBatch(TRUE); break; case 'all': - $this->prepareToExportAllContent(); + $this->exportAllBatch(); break; } - // Edit and export all entities to folder. - $this->editEntityData(); - $this->writeConfigsToFolder(); - return $this; } /** - * Prepare content to export. + * Export content in batch. + * + * @param boolean $with_references + * Indicates if export should consider referenced entities. * * @throws \Drupal\Component\Plugin\Exception\InvalidPluginDefinitionException * @throws \Drupal\Component\Plugin\Exception\PluginNotFoundException */ - private function prepareToExport() { + private function exportBatch($with_references = FALSE) { $entity_type = $this->entityTypeId; $exported_entity_ids = $this->getEntityIdsForExport(); @@ -452,33 +454,112 @@ class Exporter { $this->fileSystem->deleteRecursive($this->getFolder()); } + $export_type = $with_references ? "exportBatchWithReferences" : "exportBatchDefault"; + foreach ($exported_entity_ids as $entity_id) { - /** @var \Drupal\Core\Entity\ContentEntityInterface $entity */ - $entity = $this->entityTypeManager->getStorage($entity_type)->load($entity_id); - $exported_entity = $this->getSerializedContent($entity); - $this->addExportedEntity($exported_entity); + $operations[] = [ + [$this, $export_type], + [$entity_type, $entity_id], + ]; } + + // Set up batch information. + $batch = [ + 'title' => t('Exporting content'), + 'operations' => $operations, + 'finished' => [$this, 'exportFinished'], + ]; + + batch_set($batch); + + if (PHP_SAPI === 'cli') { + drush_backend_batch_process(); + } } /** - * Prepare content with reference to export. + * Prepares and exports a single entity to a JSON file. * - * @throws \Drupal\Component\Plugin\Exception\InvalidPluginDefinitionException - * @throws \Drupal\Component\Plugin\Exception\PluginNotFoundException - */ - private function prepareToExportWithReference() { - $entity_type = $this->entityTypeId; - $exported_entity_ids = $this->getEntityIdsForExport(); + * @param string $entity_type + * The type of the entity being exported. + * @param int $entity_id + * The ID of the entity being exported. + * @param array $context + * The batch context. + */ + public function exportBatchDefault($entity_type, $entity_id, &$context) { + // Prepare export of entity. + $entity = $this->entityTypeManager->getStorage($entity_type)->load($entity_id); + $exported_entity = $this->getSerializedContent($entity); + + // Remove or add a new fields to serialize entities data. + $entity_array = $this->serializer->decode($exported_entity, 'hal_json'); + $entity_type_object = $this->entityTypeManager->getDefinition($entity_type); + $id_key = $entity_type_object->getKey('id'); + $entity_id = $entity_array[$id_key][0]['value']; + + unset($entity_array[$entity_type_object->getKey('revision')]); + + if ($entity_type === 'user') { + $entity_array['pass'][0]['value'] = $entity->getPassword(); + } - if ($this->forceUpdate) { - $this->fileSystem->deleteRecursive($this->getFolder()); + $data = $this->serializer->serialize($entity_array, 'hal_json', [ + 'json_encode_options' => JSON_PRETTY_PRINT + ]); + + // Write serialized entity to JSON file. + $uuid = $entity->get('uuid')->value; + $entity_type_folder = "{$this->getFolder()}/{$entity_type}"; + $this->fileSystem->prepareDirectory($entity_type_folder, FileSystemInterface::CREATE_DIRECTORY); + + file_put_contents("{$entity_type_folder}/{$uuid}.json", $data); + } + + /** + * Prepares and exports a single entity to a JSON file with references. + * + * @param string $entity_type + * The type of the entity being exported. + * @param int $entity_id + * The ID of the entity being exported. + * @param array $context + * The batch context. + */ + public function exportBatchWithReferences($entity_type, $entity_id, &$context) { + // Get referenced entities. + $entity = $this->entityTypeManager->getStorage($entity_type)->load($entity_id); + + $entities = []; + if ($entity instanceof ContentEntityInterface) { + $indexed_dependencies = [$entity->uuid() => $entity]; + $entities = $this->getEntityReferencesRecursive($entity, 0, $indexed_dependencies); } - foreach ($exported_entity_ids as $entity_id) { - /** @var \Drupal\Core\Entity\ContentEntityInterface $entity */ + foreach ($entities as $uuid => $exported_entity) { + $entity_type = $exported_entity->getEntityTypeId(); + $serialized_entity = $this->getSerializedContent($exported_entity); + $entity_array = $this->serializer->decode($serialized_entity, 'hal_json'); + $entity_type_object = $this->entityTypeManager->getDefinition($entity_type); + $id_key = $entity_type_object->getKey('id'); + $entity_id = $entity_array[$id_key][0]['value']; $entity = $this->entityTypeManager->getStorage($entity_type)->load($entity_id); - $exported_entities = $this->getSerializedContentWithReferences($entity); - $this->addExportedEntity($exported_entities); + + unset($entity_array[$entity_type_object->getKey('revision')]); + + if ($entity_type === 'user') { + $entity_array['pass'][0]['value'] = $entity->getPassword(); + } + + $data = $this->serializer->serialize($entity_array, 'hal_json', [ + 'json_encode_options' => JSON_PRETTY_PRINT + ]); + + // Write serialized entity to JSON file. + $entity_type_folder = "{$this->getFolder()}/{$entity_type}"; + $this->fileSystem->prepareDirectory($entity_type_folder, FileSystemInterface::CREATE_DIRECTORY); + + file_put_contents("{$entity_type_folder}/{$uuid}.json", $data); } } @@ -488,7 +569,7 @@ class Exporter { * @throws \Drupal\Component\Plugin\Exception\InvalidPluginDefinitionException * @throws \Drupal\Component\Plugin\Exception\PluginNotFoundException */ - private function prepareToExportAllContent() { + private function exportAllBatch() { $content_entity_types = $this->deployManager->getContentEntityTypes(); if ($this->forceUpdate) { @@ -506,18 +587,43 @@ class Exporter { $entity_ids = array_values($query->execute()); foreach ($entity_ids as $entity_id) { - /** @var \Drupal\Core\Entity\ContentEntityInterface $entity */ - $entity = $this->entityTypeManager->getStorage($entity_type)->load($entity_id); - - if ($time && $entity instanceof EntityChangedInterface && $entity->getChangedTimeAcrossTranslations() < $time) { - continue; - } - - $exported_entity = $this->getSerializedContent($entity); - $this->addExportedEntity($exported_entity); + $operations[] = [ + [$this, 'exportBatchDefault'], + [$entity_type, $entity_id], + ]; } } } + + // Set up batch information. + $batch = [ + 'title' => t('Exporting content'), + 'operations' => $operations, + 'finished' => [$this, 'exportFinished'], + ]; + + batch_set($batch); + + if (PHP_SAPI === 'cli') { + drush_backend_batch_process(); + } + } + + /** + * Callback function to handle batch processing completion. + * + * @param bool $success + * Indicates whether the batch processing was successful. + */ + public function exportFinished($success, $results, $operations) { + if ($success) { + // Batch processing completed successfully. + \Drupal::messenger()->addMessage(t('Batch export completed successfully.')); + } + else { + // Batch processing encountered an error. + \Drupal::messenger()->addMessage(t('An error occurred during the batch export process.'), 'error'); + } } /** @@ -672,29 +778,6 @@ class Exporter { return $content; } - /** - * Exports a single entity and all its referenced entity. - * - * @param \Drupal\Core\Entity\ContentEntityInterface $entity - * - * @return array - * - * @throws \Drupal\Component\Plugin\Exception\InvalidPluginDefinitionException - * @throws \Drupal\Component\Plugin\Exception\PluginNotFoundException - */ - private function getSerializedContentWithReferences(ContentEntityInterface $entity) { - $indexed_dependencies = [$entity->uuid() => $entity]; - $entities = $this->getEntityReferencesRecursive($entity, 0, $indexed_dependencies); - $serialized_entities = []; - - // Serialize all entities and key them by entity TYPE and uuid. - foreach ($entities as $referenced_entity) { - $serialized_entities[$referenced_entity->getEntityTypeId()][$referenced_entity->uuid()] = $this->getSerializedContent($referenced_entity); - } - - return $serialized_entities; - } - /** * Returns all layout builder referenced blocks of an entity. * @@ -763,6 +846,8 @@ class Exporter { * Keyed array of entities indexed by entity type and ID. */ private function getEntityProcessedTextDependencies(ContentEntityInterface $entity) { + $config = $this->config->get('default_content_deploy.content_directory'); + $enabled_entity_types = $config->get('enabled_entity_types'); $entity_dependencies = []; $field_definitions = $entity->getFieldDefinitions(); @@ -789,14 +874,17 @@ class Exporter { // Iterate over all elements with a data-entity-type attribute. foreach ($xpath->query('//*[@data-entity-type]') as $node) { $entity_type = $node->getAttribute('data-entity-type'); - $uuid = $node->getAttribute('data-entity-uuid'); - - // Only add the dependency if it does not already exist. - if (!in_array($uuid, $entity_dependencies)) { - $entity_loaded_by_uuid = $this->entityTypeManager - ->getStorage($entity_type) - ->loadByProperties(['uuid' => $uuid]); - $entity_dependencies[] = reset($entity_loaded_by_uuid); + + if (in_array($entity_type, $enabled_entity_types)) { + $uuid = $node->getAttribute('data-entity-uuid'); + + // Only add the dependency if it does not already exist. + if (!in_array($uuid, $entity_dependencies)) { + $entity_loaded_by_uuid = $this->entityTypeManager + ->getStorage($entity_type) + ->loadByProperties(['uuid' => $uuid]); + $entity_dependencies[] = reset($entity_loaded_by_uuid); + } } } } @@ -843,7 +931,14 @@ class Exporter { * Keyed array of entities indexed by entity type and ID. */ private function getEntityReferencesRecursive(ContentEntityInterface $entity, $depth = 0, array &$indexed_dependencies = []) { - $entity_dependencies = $entity->referencedEntities(); + $entity_dependencies = []; + $languages = $entity->getTranslationLanguages(); + + foreach (array_keys($languages) as $langcode) { + $entity = $this->entityRepository->getTranslationFromContext($entity, $langcode); + $entity_dependencies = array_merge($entity_dependencies, $entity->referencedEntities()); + } + $entity_layout_builder_dependencies = $this->getEntityLayoutBuilderDependencies($entity); $entity_processed_text_dependencies = []; @@ -862,6 +957,13 @@ class Exporter { continue; } + // Return if entity is not in the configured referencable entity types to export. + $config = $this->config->get('default_content_deploy.content_directory'); + $enabled_entity_types = $config->get('enabled_entity_types'); + if (!in_array($dependent_entity->getEntityTypeId(), $enabled_entity_types)) { + continue; + } + // Using UUID to keep dependencies unique to prevent recursion. $key = $dependent_entity->uuid(); if (isset($indexed_dependencies[$key])) { diff --git a/src/Form/ImportForm.php b/src/Form/ImportForm.php index 6625f47..ab3c5a1 100644 --- a/src/Form/ImportForm.php +++ b/src/Form/ImportForm.php @@ -163,8 +163,6 @@ class ImportForm extends FormBase { } $this->importer->setForceOverride($force_override); - $this->importer->prepareForImport(); - $this->addResultMessage(); $this->importer->import(); } catch (\Exception $exception) { @@ -172,27 +170,4 @@ class ImportForm extends FormBase { } } - /** - * Add a message with importing results. - */ - private function addResultMessage() { - $result = $this->importer->getResult(); - $array_column = array_column($result, 'status'); - $count = array_count_values($array_column); - - $this->messenger->addMessage($this->t('Created: @count', [ - '@count' => isset($count['create']) ? $count['create'] : 0, - ])); - - $this->messenger->addMessage($this->t('Updated: @count', [ - '@count' => isset($count['update']) ? $count['update'] : 0, - ])); - - $this->messenger->addMessage($this->t('Skipped: @count', [ - '@count' => isset($count['skip']) ? $count['skip'] : 0, - ])); - - return $this; - } - } diff --git a/src/Form/SettingsForm.php b/src/Form/SettingsForm.php index fa6a93e..ae896aa 100644 --- a/src/Form/SettingsForm.php +++ b/src/Form/SettingsForm.php @@ -9,6 +9,10 @@ namespace Drupal\default_content_deploy\Form; use Drupal\Core\Form\ConfigFormBase; use Drupal\Core\Form\FormStateInterface; +use Drupal\Core\Config\ConfigFactoryInterface; +use Drupal\Core\Entity\ContentEntityTypeInterface; +use Drupal\Core\Entity\EntityTypeManagerInterface; +use Symfony\Component\DependencyInjection\ContainerInterface; /** * Settings form. @@ -20,6 +24,31 @@ class SettingsForm extends ConfigFormBase { */ const DIRECTORY = 'default_content_deploy.content_directory'; + /** + * The Entity type manager. + * + * @var \Drupal\Core\Entity\EntityTypeManagerInterface + */ + protected $entityTypeManager; + + /** + * {@inheritdoc} + */ + public function __construct(ConfigFactoryInterface $config_factory, EntityTypeManagerInterface $entity_type_manager) { + parent::__construct($config_factory); + $this->entityTypeManager = $entity_type_manager; + } + + /** + * {@inheritdoc} + */ + public static function create(ContainerInterface $container) { + return new static( + $container->get('config.factory'), + $container->get('entity_type.manager'), + ); + } + /** * {@inheritdoc} */ @@ -56,6 +85,42 @@ class SettingsForm extends ConfigFormBase { '#description' => 'If selected, embedded entities within processed text fields will be included in the export.', ]; + $form['support_old_content'] = [ + '#type' => 'checkbox', + '#title' => $this->t('Support old content'), + '#default_value' => $config->get('support_old_content'), + '#description' => 'If selected, the import process with run additional processes to ensure reference integrity for older content.', + ]; + + $all_entity_types = $this->entityTypeManager->getDefinitions(); + $content_entity_types = []; + + // Filter the entity types. + /** @var \Drupal\Core\Entity\EntityTypeInterface[] $entity_type_options */ + foreach ($all_entity_types as $entity_type) { + if (($entity_type instanceof ContentEntityTypeInterface)) { + $content_entity_types[$entity_type->id()] = $entity_type->getLabel(); + } + } + + // Entity types. + $form['enabled_entity_types'] = [ + '#type' => 'details', + '#open' => FALSE, + '#title' => $this->t('Enabled entity types'), + '#description' => $this->t('Check which entity types should be exported by reference.'), + '#tree' => TRUE, + ]; + + $form['enabled_entity_types']['entity_types'] = [ + '#type' => 'checkboxes', + '#title' => $this->t('Enabled entity types'), + '#options' => $content_entity_types, + // If no custom settings exist, content entities are enabled by default. + '#default_value' => $config->get('enabled_entity_types') ?: array_keys($content_entity_types), + '#required' => TRUE, + ]; + return parent::buildForm($form, $form_state); } @@ -65,7 +130,9 @@ class SettingsForm extends ConfigFormBase { public function submitForm(array &$form, FormStateInterface $form_state) { $this->configFactory->getEditable(static::DIRECTORY) ->set('content_directory', $form_state->getValue('content_directory')) + ->set('enabled_entity_types', array_values(array_filter($form_state->getValue('enabled_entity_types')['entity_types']))) ->set('text_dependencies', $form_state->getValue('text_dependencies')) + ->set('support_old_content', $form_state->getValue('support_old_content')) ->save(); parent::submitForm($form, $form_state); diff --git a/src/Importer.php b/src/Importer.php index 3ef0a69..8e80625 100644 --- a/src/Importer.php +++ b/src/Importer.php @@ -4,10 +4,13 @@ namespace Drupal\default_content_deploy; use Drupal\Component\EventDispatcher\ContainerAwareEventDispatcher; use Drupal\Core\Cache\CacheBackendInterface; +use Drupal\Core\Config\ConfigFactoryInterface; use Drupal\Core\Database\Connection; +use Drupal\Core\DependencyInjection\DependencySerializationTrait; use Drupal\Core\Entity\EntityChangedInterface; use Drupal\Core\Entity\EntityRepositoryInterface; use Drupal\Core\Entity\EntityTypeManagerInterface; +use Drupal\Core\KeyValueStore\KeyValueFactory; use Drupal\Core\Session\AccountSwitcherInterface; use Drupal\default_content_deploy\Event\PreSaveEntityEvent; use Drupal\hal\LinkManager\LinkManagerInterface; @@ -15,14 +18,12 @@ use Rogervila\ArrayDiffMultidimensional; use Symfony\Component\Serializer\Serializer; /** - * A service for handling import of default content. - * - * The importContent() method is almost duplicate of - * \Drupal\default_content\Importer::importContent with injected code for - * content update. We are waiting for better DC code structure in a future. + * A service for handling the import of content using batch API. */ class Importer { + use DependencySerializationTrait; + /** * Deploy manager. * @@ -31,11 +32,11 @@ class Importer { protected $deployManager; /** - * Scanned files. + * The key-value store service. * - * @var object[] + * @var \Drupal\Core\KeyValueStore\KeyValueFactory */ - private $files; + protected $keyValueStorage; /** * Directory to import. @@ -72,13 +73,6 @@ class Importer { */ protected $cache; - /** - * Memoization for references that have already been discovered. - * - * @var array - */ - protected $discoveredReferences = []; - /** * The serializer service. * @@ -127,19 +121,18 @@ class Importer { protected $eventDispatcher; /** + * The batch context. + * * @var array */ - protected $oldEntityIdLookup = []; - - /** - * @var array - */ - protected $entityIdLookup = []; + protected $context; /** - * @var array + * The config factory. + * + * @var \Drupal\Core\Config\ConfigFactoryInterface */ - protected $newUuids = []; + protected $config; /** * Constructs the default content deploy manager. @@ -162,8 +155,12 @@ class Importer { * Database connection. * @param \Drupal\Component\EventDispatcher\ContainerAwareEventDispatcher $event_dispatcher * The event dispatcher. + * @param \Drupal\Core\KeyValueStore\KeyValueFactory $key_value_factory + * The key value factory. + * @param \Drupal\Core\Config\ConfigFactoryInterface $config + * The config factory. */ - public function __construct(Serializer $serializer, EntityTypeManagerInterface $entity_type_manager, LinkManagerInterface $link_manager, AccountSwitcherInterface $account_switcher, DeployManager $deploy_manager, EntityRepositoryInterface $entity_repository, CacheBackendInterface $cache, Exporter $exporter, Connection $database, ContainerAwareEventDispatcher $event_dispatcher) { + public function __construct(Serializer $serializer, EntityTypeManagerInterface $entity_type_manager, LinkManagerInterface $link_manager, AccountSwitcherInterface $account_switcher, DeployManager $deploy_manager, EntityRepositoryInterface $entity_repository, CacheBackendInterface $cache, Exporter $exporter, Connection $database, ContainerAwareEventDispatcher $event_dispatcher, KeyValueFactory $key_value_factory, ConfigFactoryInterface $config) { $this->serializer = $serializer; $this->entityTypeManager = $entity_type_manager; $this->linkManager = $link_manager; @@ -174,6 +171,8 @@ class Importer { $this->exporter = $exporter; $this->database = $database; $this->eventDispatcher = $event_dispatcher; + $this->keyValueStorage = $key_value_factory; + $this->config = $config; } /** @@ -228,31 +227,6 @@ class Importer { return $this->dataToImport; } - /** - * Import data from JSON and create new entities, or update existing. - * - * @return $this - * - * @throws \Drupal\Component\Plugin\Exception\InvalidPluginDefinitionException - * @throws \Drupal\Component\Plugin\Exception\PluginNotFoundException - * @throws \Exception - */ - public function prepareForImport() { - // @todo remove because of changes in core >= 9.2 - $this->cache->delete('hal:links:relations'); - $this->files = $this->scan($this->getFolder()); - - foreach ($this->files as $file) { - $uuid = str_replace('.json', '', $file->name); - - if (!isset($this->dataToImport[$uuid])) { - $this->decodeFile($file); - } - } - - return $this; - } - /** * Returns a list of file objects. * @@ -265,14 +239,13 @@ class Importer { public function scan($directory) { // Use Unix paths regardless of platform, skip dot directories, follow // symlinks (to allow extensions to be linked from elsewhere), and return - // the RecursiveDirectoryIterator instance to have access to getSubPath(), - // since SplFileInfo does not support relative paths. + // the RecursiveDirectoryIterator instance. $flags = \FilesystemIterator::UNIX_PATHS; $flags |= \FilesystemIterator::SKIP_DOTS; $flags |= \FilesystemIterator::CURRENT_AS_SELF; $directory_iterator = new \RecursiveDirectoryIterator($directory, $flags); $iterator = new \RecursiveIteratorIterator($directory_iterator); - $files = []; + $files = $other_files = $alias_files = []; /* @var \SplFileInfo $file_info */ foreach ($iterator as $file_info) { @@ -284,148 +257,139 @@ class Importer { $file = new \stdClass(); $file->name = $file_info->getFilename(); $file->uri = $file_info->getPathname(); - $files[$file->uri] = $file; + + // Put url_alias files at the end of the array to ensure + // nodes have already been imported. + if (strpos($file->uri, '/url_alias/') !== false) { + $alias_files[$file->uri] = $file; + } + else { + $other_files[$file->uri] = $file; + } } + $files = array_merge($other_files, $alias_files); + return $files; } /** - * Import to entity. + * Import data from JSON and create new entities, or update existing. * - * @throws \Drupal\Component\Plugin\Exception\InvalidPluginDefinitionException - * @throws \Drupal\Component\Plugin\Exception\PluginNotFoundException - * @throws \Drupal\Core\Entity\EntityStorageException + * @return $this */ public function import() { - $files = $this->dataToImport; + // Get a list of all files to import. + $files = $this->scan($this->getFolder()); - if (PHP_SAPI === 'cli') { - $root_user = $this->entityTypeManager->getStorage('user')->load(1); - $this->accountSwitcher->switchTo($root_user); + // Save files to KeyValueStoreInterface to ensure they are not + // stored in the class and subsequently added to the queue database table. + $this->setFilesToImport($files); + + // Process files in batches. + $operations = []; + $total = count($files); + $current = 1; + + if ($total == 0) { + \Drupal::messenger()->addMessage(t('Nothing to import.')); + return; } - // All entities with entity references will be imported two times to ensure - // that all entity references are present and valid. Path aliases will be - // imported last to have a chance to rewrite them to the new ids of newly - // created entities. - for ($i = 0; $i <= 2; $i++) { - foreach ($files as $uuid => &$file) { - $entity_type = $file['entity_type_id']; - if ($file['status'] !== 'skip') { - if ( - ($i !== 2 && $entity_type === 'path_alias') || - ($i === 2 && $entity_type !== 'path_alias') - ) { - continue; - } - $this->linkManager->setLinkDomain($this->getLinkDomain($file)); - $class = $this->entityTypeManager->getDefinition($entity_type)->getClass(); - $needs_second_run = $this->preDenormalize($file, $entity_type); - - /** @var \Drupal\Core\Entity\ContentEntityInterface $entity */ - $entity = $this->serializer->denormalize($file['data'], $class, 'hal_json', ['request_method' => 'POST']); - $this->eventDispatcher->dispatch(new PreSaveEntityEvent($entity, $file['data'])); - $entity->enforceIsNew($file['is_new']); - $entity->save(); - $this->entityIdLookup[$uuid] = $entity->id(); - - if ($file['is_new']) { - $this->newUuids[] = $uuid; - } + // Initialize progress tracking here. + if (!isset($this->context['sandbox']['progress'])) { + $this->context['sandbox']['progress'] = 0; + $this->context['sandbox']['total'] = $total; + } - if ($entity_type === 'user') { - // Workaround: store the hashed password directly in the database - // and avoid the entity API which doesn't provide support for - // setting password hashes directly. - $hashed_pass = $file['data']['pass'][0]['value'] ?? FALSE; - if ($hashed_pass) { - $this->database->update('users_field_data') - ->fields([ - 'pass' => $hashed_pass, - ]) - ->condition('uid', $entity->id(), '=') - ->execute(); - } - } + foreach ($files as $file) { + $operations[] = [ + [$this, 'processFile'], + [$file, $current, $total], + ]; - if ((!$needs_second_run && empty($file['references'])) || $i === 1) { - // Don't handle entities without references twice. Don't handle - // entities with references again in the third run for path aliases. - unset($files[$uuid]); - } - else { - // In the second run new entities should be updated. - $file['status'] = 'update'; - $file['is_new'] = FALSE; - $file['data'][$file['key_id']][0]['value'] = $entity->id(); - } - } - elseif ($i === 0) { - // Get the entity ID of a skipped referenced item in the first run to - // enable a target ID correction in referencing entities in the second - // and third run. - $entity = $this->entityRepository->loadEntityByUuid($entity_type, $uuid); - $this->entityIdLookup[$uuid] = $entity->id(); - } - } - unset($file); + $current++; } - // @todo is this still needed? - $this->linkManager->setLinkDomain(FALSE); + $batch = [ + 'title' => t('Importing Content'), + 'operations' => $operations, + 'finished' => [$this, 'importFinished'], + ]; + + batch_set($batch); if (PHP_SAPI === 'cli') { - $this->accountSwitcher->switchBack(); - } - } + // Switch to user 1 is running import from CLI. + $root_user = $this->entityTypeManager->getStorage('user')->load(1); + $this->accountSwitcher->switchTo($root_user); - /** - * Gets url from file for set to Link manager. - * - * @param array $file - */ - protected function getLinkDomain($file) { - $link = $file['data']['_links']['type']['href']; - $url_data = parse_url($link); - $host = "{$url_data['scheme']}://{$url_data['host']}"; - return (!isset($url_data['port'])) ? $host : "{$host}:{$url_data['port']}"; + // Process hatch with drush. + drush_backend_batch_process(); + } } /** - * Prepare file to import. + * Prepare file for import. * * @param $file + * The file object to use for import. + * @param $current + * Indicates progress of the batch operations. + * @param $total + * Total number of batch operations. + * @param array &$context + * Reference to an array that stores the context of the batch process for status updates. * * @return $this * - * @throws \Drupal\Component\Plugin\Exception\InvalidPluginDefinitionException - * @throws \Drupal\Component\Plugin\Exception\PluginNotFoundException * @throws \Exception */ - protected function decodeFile($file) { - // Check that this file has not been decoded already. - if (array_key_exists($file->name, $this->discoveredReferences)) { + public function processFile($file, $current, $total, &$context) { + // Set files array into $context so we can access across batch operations. + if (empty($context['results']['files'])) { + $config = $this->config->get('default_content_deploy.content_directory'); + $support_old_content = $config->get('support_old_content'); + + $context['results']['files'] = $this->getFilesToImport(); + $context['results']['start'] = microtime(TRUE); + $context['results']['support_old_content'] = $support_old_content; + // Delete KeyValueStorage values since they are now stored in $context. + $this->deleteFilesToImport(); + } + + // Check that the current file has been processed already. + if (!empty($context['results']['processed']) && in_array($file->name, $context['results']['processed'])) { + $context['message'] = t('Importing entity @current of @total (@time)', [ + '@current' => $current, + '@total' => $total, + '@time' => $this->getElapsedTime($context['results']['start']), + ]); + return $this; } - // Get parsed data. + // Get parsed file contents. $parsed_data = file_get_contents($file->uri); - // Decode. + // Try to decode the parsed data. try { $decode = $this->serializer->decode($parsed_data, 'hal_json'); } catch (\Exception $e) { throw new \RuntimeException(sprintf('Unable to decode %s', $file->uri), $e->getCode(), $e); } - $references = $this->getReferences($decode); + + // Get references for current entity. + $references = $this->getReferences($decode, $context['results']['files']); // Record that we have checked references of current file. - $this->discoveredReferences[$file->name] = $file; + $context['results']['processed'][] = $file->name; + + // If there are references then process them recursively. if ($references) { foreach ($references as $reference) { - $this->decodeFile($reference); + $this->processFile($reference, $current, $total, $context); } } @@ -435,43 +399,55 @@ class Importer { 'data' => $decode, 'entity_type_id' => $this->getEntityTypeByLink($link), 'references' => $references, + 'filename' => $file->name, ]; + $this->preAddToImport($data_to_import, $context); + + // Import entity. + $this->importEntity($data_to_import, $total, $current, $context); - $this->preAddToImport($data_to_import); - $this->addToImport($data_to_import); + // Pass export data to results. + $uuid = $data_to_import['data']['uuid'][0]['value']; + $context['results']['data_to_import'][$uuid]['status'] = $data_to_import['status']; + $context['message'] = t('Importing entity @current of @total (@time)', [ + '@current' => $current, + '@total' => $total, + '@time' => $this->getElapsedTime($context['results']['start']), + ]); return $this; } /** - * Here we can edit data`s value before importing. + * Manipulate entity data before importing. * * @param $data + * Entity data to manipulate before import. + * @param array &$context + * Reference to an array that stores the context of the batch process for status updates. * * @return $this - * - * @throws \Drupal\Component\Plugin\Exception\PluginNotFoundException - * @throws \Drupal\Core\Entity\EntityStorageException - * @throws \Drupal\Component\Plugin\Exception\InvalidPluginDefinitionException */ - protected function preAddToImport(&$data) { + protected function preAddToImport(&$data, &$context) { $decode = $data['data']; $uuid = $decode['uuid'][0]['value']; $entity_type_id = $data['entity_type_id']; - /** @var \Drupal\Core\Entity\ContentEntityInterface $entity */ - $entity = $this->entityRepository->loadEntityByUuid($entity_type_id, $uuid); $entity_type_object = $this->entityTypeManager->getDefinition($entity_type_id); - // Keys of entity. + // Get keys for entity. $key_id = $entity_type_object->getKey('id'); $key_revision_id = $entity_type_object->getKey('revision'); // Some old exports don't have the entity ID. if (isset($decode[$key_id][0]['value'])) { - $this->oldEntityIdLookup[$entity_type_id][$decode[$key_id][0]['value']][] = $uuid; + $context['results']['old_entity_lookup'][$decode[$key_id][0]['value']] = $uuid; } + // Try to load the entity. + $entity = $this->entityRepository->loadEntityByUuid($entity_type_id, $uuid); + if ($entity) { + $data['entity_id'] = $entity->id(); $is_new = FALSE; $status = 'update'; @@ -499,12 +475,17 @@ class Importer { elseif (!$this->forceOverride) { $this->linkManager->setLinkDomain($this->getLinkDomain($data)); $current_entity_decoded = $this->serializer->decode($this->exporter->getSerializedContent($entity), 'hal_json'); + + if ($entity_type_id == 'path_alias') { + // Make sure the path matches to accound for updatePathAliasTargetId(). + $decode['path'] = $current_entity_decoded['path']; + $decode['_links']['type']['href'] = $current_entity_decoded['_links']['type']['href']; + } + $diff = ArrayDiffMultidimensional::looseComparison($decode, $current_entity_decoded); if (!$diff) { $status = 'skip'; } - // @todo is this still needed? - $this->linkManager->setLinkDomain(FALSE); } } else { @@ -516,8 +497,7 @@ class Importer { } // @see path_entity_base_field_info(). - // @todo offer an event to let third party modules register their content - // types. + // @todo offer an event to register other content types. if (in_array($entity_type_id, ['taxonomy_term', 'node', 'media'])) { unset($decode['path']); } @@ -534,47 +514,169 @@ class Importer { } /** - * This event is triggered before decoding to an entity. + * Imports a single entity as part of the batch process. * - * @param $file + * @param array $data + * Array containing the entity data to be imported. + * @param int $total + * Total number of entities to be processed in this batch for progress tracking. + * @param int $current + * Index of the current entity being processed in the total batch. + * @param array &$context + * Reference to an array that stores the context of the batch process for status updates. * - * @throws \Drupal\Core\Entity\EntityStorageException + * @return $this */ - protected function preDenormalize(&$file, $entity_type) { - $needs_second_run = FALSE; + public function importEntity($data, $total, $current, &$context) { + $uuid = $data['data']['uuid'][0]['value']; + $entity_status = $data['status']; + + if ($entity_status == 'skip') { + // Skip this record, update ID lookup. + $context['results']['entity_lookup'][$uuid] = $data['entity_id']; + } + else { + $entity_type_id = $data['entity_type_id']; + $entity_type_object = $this->entityTypeManager->getDefinition($entity_type_id); + + $this->linkManager->setLinkDomain($this->getLinkDomain($data)); + $class = $entity_type_object->getClass(); + + // Processes to run against older content. + if ($context['results']['support_old_content']) { + // Update internal links. + $this->updateInternalLinks($data, $context); + } + + switch ($entity_type_id) { + case 'path_alias': + // Update url_alias reference to entity. + $this->updatePathAliasTargetId($data, $context); + break; - $this->updateTargetRevisionId($file['data']); - $needs_second_run |= $this->updateInternalLinks($file['data']); + case 'paragraph': + // Update target revision IDs. + $this->updateTargetRevisionId($data); + break; + } - if ($entity_type === 'path_alias') { - $this->updatePathAliasTargetId($file['data']); + // Prepare the entity for save. + $entity = $this->serializer->denormalize($data['data'], $class, 'hal_json', ['request_method' => 'POST']); + $this->eventDispatcher->dispatch(new PreSaveEntityEvent($entity, $data)); + + // Save entity. + $entity->enforceIsNew($data['is_new']); + $entity->save(); + + // Update entity ID lookup with the UUID and entity ID. + $context['results']['entity_lookup'][$uuid] = $entity->id(); + + if ($entity_type_id === 'user') { + // Workaround: store the hashed password directly in the database + // and avoid the entity API which doesn't provide support for + // setting password hashes directly. + $hashed_pass = $data['pass'][0]['value'] ?? FALSE; + if ($hashed_pass) { + $this->database->update('users_field_data') + ->fields([ + 'pass' => $hashed_pass, + ]) + ->condition('uid', $entity->id(), '=') + ->execute(); + } + } } - return $needs_second_run; + // Provide update progress. + $context['message'] = t('Importing entity @current of @total', ['@current' => $current, '@total' => $total]); + + return $this; } /** - * Adding prepared data for import. + * Callback function to handle batch completion. * - * @param $data + * @param bool $success + * Indicates whether the batch processing was successful. + * @param array $results + * The stored results for batch processing. + * @param array $operations + * An array of the operations that were run during the batch process. + */ + public function importFinished($success, $results, $operations) { + if (PHP_SAPI === 'cli') { + // If using CLI, switch back from user 1. + $this->accountSwitcher->switchBack(); + } + + if ($success) { + // Get elapsed time for the overall batch process. + $elapsed_time = $this->getElapsedTime($results['start']); + + // Batch processing completed successfully. + \Drupal::messenger()->addMessage(t('Batch import completed successfully in') . ' ' . $elapsed_time); + + // Output result counts. + $results = !empty($results['data_to_import']) ? $results['data_to_import'] : []; + $array_column = array_column($results, 'status'); + $count = array_count_values($array_column); + + \Drupal::messenger()->addMessage(t('Created: @count', [ + '@count' => isset($count['create']) ? $count['create'] : 0, + ])); + + \Drupal::messenger()->addMessage(t('Updated: @count', [ + '@count' => isset($count['update']) ? $count['update'] : 0, + ])); + + \Drupal::messenger()->addMessage(t('Skipped: @count', [ + '@count' => isset($count['skip']) ? $count['skip'] : 0, + ])); + } + else { + // Batch processing encountered an error. + \Drupal::messenger()->addMessage(t('An error occurred during the batch import process.'), 'error'); + } + } + + /** + * Calculates and formats the elapsed time. * - * @return $this + * @param float $start + * The start time of the overall batch process. + * + * @return string + * The formatted elapsed time in minutes. */ - protected function addToImport($data) { - $uuid = $data['data']['uuid'][0]['value']; - $this->dataToImport[$uuid] = $data; + public function getElapsedTime($start) { + $end = microtime(TRUE); + $diff = $end - $start; + $elapsed_time = number_format($diff / 60, 2) . ' ' . t('minutes'); - return $this; + return $elapsed_time; + } + + /** + * Gets url from file for set to Link manager. + * + * @param array $file + */ + protected function getLinkDomain($file) { + $link = $file['data']['_links']['type']['href']; + $url_data = parse_url($link); + $host = "{$url_data['scheme']}://{$url_data['host']}"; + return (!isset($url_data['port'])) ? $host : "{$host}:{$url_data['port']}"; } /** * Get all reference by entity array content. * * @param array $content + * @param array $files * * @return array */ - private function getReferences(array $content) { + private function getReferences(array $content, $files) { $references = []; if (isset($content['_embedded'])) { @@ -582,10 +684,10 @@ class Importer { foreach ($link as $reference) { if ($reference) { $uuid = $reference['uuid'][0]['value']; - $path = $this->getPathToFileByName($uuid); + $path = $this->getPathToFileByName($uuid, $files); if ($path) { - $references[$uuid] = $this->files[$path]; + $references[$uuid] = $files[$path]; } } } @@ -598,19 +700,54 @@ class Importer { /** * Get path to file by Name. * - * @param $name + * @param string $name + * @param array $files * * @return false|int|string */ - private function getPathToFileByName($name) { - $array_column = array_column($this->files, 'name', 'uri'); + private function getPathToFileByName($name, $files) { + $array_column = array_column($files, 'name', 'uri'); return array_search($name . '.json', $array_column); } + /** + * Get reference to all JSON files for import. + * + * @return array|NULL + */ + private function getFilesToImport() { + // Get files from keyValueStorage. + $files = $this->keyValueStorage->get('dcd_batch_files')->get('dcd_batch_files'); + + return $files; + } + + /** + * Set reference for all JSON files to import. + * + * @param array $files + * + * @return array|NULL + */ + private function setFilesToImport($files) { + // Get files from keyValueStorage. + $this->keyValueStorage->get('dcd_batch_files')->set('dcd_batch_files', $files); + } + + /** + * Delete reference to JSON files for import. + * + * @return array|NULL + */ + private function deleteFilesToImport() { + // Get files from keyValueStorage. + $this->keyValueStorage->get('dcd_batch_files')->delete('dcd_batch_files'); + } + /** * Get Entity type ID by link. * - * @param $link + * @param string $link * * @return string|string[] */ @@ -637,8 +774,6 @@ class Importer { * @param $decode * * @return $this - * - * @throws \Drupal\Core\Entity\EntityStorageException */ private function updateTargetRevisionId(&$decode) { if (isset($decode['_embedded'])) { @@ -670,30 +805,25 @@ class Importer { * @param array $decode * The decoded entity. * - * @return bool - * TRUE if the entity contains an internal link field. + * @return $this */ - private function updateInternalLinks(array &$decode): bool { - $has_internal_links = FALSE; - - foreach ($decode as $field => $items) { + private function updateInternalLinks(array &$decode, $context) { + foreach ($decode['data'] as $field => $items) { foreach ($items as $index => $item) { foreach ($item as $name => $value) { if ('uri' === $name) { if (str_starts_with($value, 'internal:')) { - $decode[$field][$index][$name] = 'internal:' . $this->getUpdatedInternalPath(str_replace('internal:', '', $value)); - $has_internal_links = TRUE; + $decode['data'][$field][$index][$name] = 'internal:' . $this->getUpdatedInternalPath(str_replace('internal:', '', $value), $context); } elseif (str_starts_with($value, 'entity:')) { - $decode[$field][$index][$name] = 'entity:' . trim($this->getUpdatedInternalPath(str_replace('entity:', '/', $value), $item['target_uuid'] ?? NULL), '/'); - $has_internal_links = TRUE; + $decode['data'][$field][$index][$name] = 'entity:' . trim($this->getUpdatedInternalPath(str_replace('entity:', '/', $value), $context, $item['target_uuid'] ?? NULL), '/'); } } } } } - return $has_internal_links; + return $this; } /** @@ -702,9 +832,9 @@ class Importer { * @param array $decode * The decoded entity. */ - private function updatePathAliasTargetId(array &$decode) { - if ($path = $decode['path'][0]['value'] ?? NULL) { - $decode['path'][0]['value'] = $this->getUpdatedInternalPath($path); + private function updatePathAliasTargetId(array &$decode, $context) { + if ($path = $decode['date']['path'][0]['value'] ?? NULL) { + $decode['data']['path'][0]['value'] = $this->getUpdatedInternalPath($path, $context); } } @@ -713,39 +843,32 @@ class Importer { * * @param string $path * The path from exported content. + * @param string $uuid + * The UUID of the existing reference. * * @return string * The updated path. - * - * @throws \InvalidArgumentException */ - private function getUpdatedInternalPath($path, $uuid = NULL): string { + private function getUpdatedInternalPath($path, $context, $uuid = NULL): string { + // The regex is designed to match URLs or paths where the structure consists of: + // - A first segment that is alphanumeric (including underscores), + // - Followed by a second segment that is purely numeric, + // - Optionally followed by path segments, query parameters, or fragment identifiers. + // @todo this logic will not work with taxonomy_term entities /taxonomy/term/[ID] if (preg_match('@^/(\w+)/(\d+)([/?#].*|)$@', $path, $matches)) { - $entity_type_id = str_replace('_', '/', $matches[1]); if (!$uuid) { - $uuids = $this->oldEntityIdLookup[$entity_type_id][$matches[2]] ?? []; - if (count($uuids) > 1) { - $new_uuid = FALSE; - foreach ($this->oldEntityIdLookup[$entity_type_id][$matches[2]] as $uuid_candidate) { - if (in_array($uuid_candidate, $this->newUuids, TRUE)) { - // If an internal path that needs to be updated or created targets - // an entity ID of new entity, this one is preferred. - $uuid = $uuid_candidate; - $new_uuid = TRUE; - break; - } - $new_uuid = FALSE; - } - if (!$new_uuid) { - throw new \InvalidArgumentException(sprintf('Multiple UUIDs (%s) share the same entity ID %d.', implode(', ', $this->oldEntityIdLookup[$entity_type_id][$matches[2]]), $matches[2])); - } - } - elseif (!empty($uuids)) { - $uuid = reset($uuids); + $oldEntityIdLookup = $context['results']['old_entity_lookup']; + $uuid_lookup = $oldEntityIdLookup[$matches[2]] ?? []; + + if (!empty($uuid_lookup)) { + $uuid = $uuid_lookup; } } + if ($uuid) { - if ($id = $this->entityIdLookup[$uuid] ?? NULL) { + $entityIdLookup = $context['results']['entity_lookup']; + + if ($id = $entityIdLookup[$uuid] ?? NULL) { return '/' . $matches[1] . '/' . $id . $matches[3]; } }