diff --git a/core/modules/file/migration_templates/d6_file.yml b/core/modules/file/migration_templates/d6_file.yml index a4693cf..e60ab62 100644 --- a/core/modules/file/migration_templates/d6_file.yml +++ b/core/modules/file/migration_templates/d6_file.yml @@ -9,13 +9,27 @@ source: process: fid: fid filename: filename - uri: + source_full_path: + - + plugin: concat + delimiter: / + source: + - constants/source_base_path + - filepath + - + plugin: urlencode + destination_full_path: plugin: file_uri source: - filepath - file_directory_path - temp_directory_path - is_public + uri: + plugin: file_copy + source: + - '@source_full_path' + - '@destination_full_path' filemime: filemime filesize: filesize status: status @@ -23,4 +37,3 @@ process: uid: uid destination: plugin: entity:file - urlencode: true diff --git a/core/modules/file/migration_templates/d7_file.yml b/core/modules/file/migration_templates/d7_file.yml index b10cca6..42fed73 100644 --- a/core/modules/file/migration_templates/d7_file.yml +++ b/core/modules/file/migration_templates/d7_file.yml @@ -9,7 +9,21 @@ source: process: fid: fid filename: filename - uri: uri + source_full_path: + - + plugin: concat + delimiter: / + source: + - constants/source_base_path + - filepath + - + plugin: urlencode + uri: + plugin: file_copy + source: + - '@source_full_path' + - uri + urlencode: true filemime: filemime # filesize is dynamically computed when file entities are saved, so there is # no point in migrating it. @@ -22,5 +36,3 @@ process: uid: uid destination: plugin: entity:file - source_path_property: filepath - urlencode: true diff --git a/core/modules/file/src/Plugin/migrate/destination/EntityFile.php b/core/modules/file/src/Plugin/migrate/destination/EntityFile.php index 981706f..8b34b05 100644 --- a/core/modules/file/src/Plugin/migrate/destination/EntityFile.php +++ b/core/modules/file/src/Plugin/migrate/destination/EntityFile.php @@ -27,52 +27,6 @@ class EntityFile extends EntityContentBase { /** - * @var \Drupal\Core\StreamWrapper\StreamWrapperManagerInterface - */ - protected $streamWrapperManager; - - /** - * @var \Drupal\Core\File\FileSystemInterface - */ - protected $fileSystem; - - /** - * {@inheritdoc} - */ - public function __construct(array $configuration, $plugin_id, $plugin_definition, MigrationInterface $migration, EntityStorageInterface $storage, array $bundles, EntityManagerInterface $entity_manager, FieldTypePluginManagerInterface $field_type_manager, StreamWrapperManagerInterface $stream_wrappers, FileSystemInterface $file_system) { - $configuration += array( - 'source_base_path' => '', - 'source_path_property' => 'filepath', - 'destination_path_property' => 'uri', - 'move' => FALSE, - 'urlencode' => FALSE, - ); - parent::__construct($configuration, $plugin_id, $plugin_definition, $migration, $storage, $bundles, $entity_manager, $field_type_manager); - - $this->streamWrapperManager = $stream_wrappers; - $this->fileSystem = $file_system; - } - - /** - * {@inheritdoc} - */ - public static function create(ContainerInterface $container, array $configuration, $plugin_id, $plugin_definition, MigrationInterface $migration = NULL) { - $entity_type = static::getEntityTypeId($plugin_id); - return new static( - $configuration, - $plugin_id, - $plugin_definition, - $migration, - $container->get('entity.manager')->getStorage($entity_type), - array_keys($container->get('entity.manager')->getBundleInfo($entity_type)), - $container->get('entity.manager'), - $container->get('plugin.manager.field.field_type'), - $container->get('stream_wrapper_manager'), - $container->get('file_system') - ); - } - - /** * {@inheritdoc} */ protected function getEntity(Row $row, array $old_destination_id_values) { @@ -82,7 +36,9 @@ protected function getEntity(Row $row, array $old_destination_id_values) { return parent::getEntity($row, $old_destination_id_values); } - $destination = $row->getDestinationProperty($this->configuration['destination_path_property']); + // By default the entity key (fid) would be used, but we want to make sure + // we're loading the matching uri. + $destination = $row->getDestinationProperty('uri'); $entity = $this->storage->loadByProperties(['uri' => $destination]); if ($entity) { return reset($entity); @@ -95,181 +51,6 @@ protected function getEntity(Row $row, array $old_destination_id_values) { /** * {@inheritdoc} */ - public function import(Row $row, array $old_destination_id_values = array()) { - // For stub rows, there is no real file to deal with, let the stubbing - // process create the stub entity. - if ($row->isStub()) { - return parent::import($row, $old_destination_id_values); - } - - $file = $row->getSourceProperty($this->configuration['source_path_property']); - $destination = $row->getDestinationProperty($this->configuration['destination_path_property']); - $source = $this->configuration['source_base_path'] . $file; - - // Ensure the source file exists, if it's a local URI or path. - if ($this->isLocalUri($source) && !file_exists($source)) { - throw new MigrateException("File '$source' does not exist."); - } - - // If the start and end file is exactly the same, there is nothing to do. - if ($this->isLocationUnchanged($source, $destination)) { - return parent::import($row, $old_destination_id_values); - } - - $replace = $this->getOverwriteMode($row); - $success = $this->writeFile($source, $destination, $replace); - if (!$success) { - $dir = $this->getDirectory($destination); - if (file_prepare_directory($dir, FILE_CREATE_DIRECTORY)) { - $success = $this->writeFile($source, $destination, $replace); - } - else { - throw new MigrateException("Could not create directory '$dir'"); - } - } - - if ($success) { - return parent::import($row, $old_destination_id_values); - } - else { - throw new MigrateException("File $source could not be copied to $destination."); - } - } - - /** - * Tries to move or copy a file. - * - * @param string $source - * The source path or URI. - * @param string $destination - * The destination path or URI. - * @param int $replace - * (optional) FILE_EXISTS_REPLACE (default) or FILE_EXISTS_RENAME. - * - * @return bool - * TRUE on success, FALSE on failure. - */ - protected function writeFile($source, $destination, $replace = FILE_EXISTS_REPLACE) { - if ($this->configuration['move']) { - return (boolean) file_unmanaged_move($source, $destination, $replace); - } - else { - $destination = file_destination($destination, $replace); - $source = $this->urlencode($source); - return @copy($source, $destination); - } - } - - /** - * Determines how to handle file conflicts. - * - * @param \Drupal\migrate\Row $row - * - * @return int - * Either FILE_EXISTS_REPLACE (default) or FILE_EXISTS_RENAME, depending - * on the current configuration. - */ - protected function getOverwriteMode(Row $row) { - if (!empty($this->configuration['rename'])) { - $entity_id = $row->getDestinationProperty($this->getKey('id')); - if ($entity_id && ($entity = $this->storage->load($entity_id))) { - return FILE_EXISTS_RENAME; - } - } - return FILE_EXISTS_REPLACE; - } - - /** - * Returns the directory component of a URI or path. - * - * For URIs like public://foo.txt, the full physical path of public:// - * will be returned, since a scheme by itself will trip up certain file - * API functions (such as file_prepare_directory()). - * - * @param string $uri - * The URI or path. - * - * @return string|false - * The directory component of the path or URI, or FALSE if it could not - * be determined. - */ - protected function getDirectory($uri) { - $dir = $this->fileSystem->dirname($uri); - if (substr($dir, -3) == '://') { - return $this->fileSystem->realpath($dir); - } - else { - return $dir; - } - } - - /** - * Returns if the source and destination URIs represent identical paths. - * If either URI is a remote stream, will return FALSE. - * - * @param string $source - * The source URI. - * @param string $destination - * The destination URI. - * - * @return bool - * TRUE if the source and destination URIs refer to the same physical path, - * otherwise FALSE. - */ - protected function isLocationUnchanged($source, $destination) { - if ($this->isLocalUri($source) && $this->isLocalUri($destination)) { - return $this->fileSystem->realpath($source) === $this->fileSystem->realpath($destination); - } - else { - return FALSE; - } - } - - /** - * Returns if the given URI or path is considered local. - * - * A URI or path is considered local if it either has no scheme component, - * or the scheme is implemented by a stream wrapper which extends - * \Drupal\Core\StreamWrapper\LocalStream. - * - * @param string $uri - * The URI or path to test. - * - * @return bool - */ - protected function isLocalUri($uri) { - $scheme = $this->fileSystem->uriScheme($uri); - return $scheme === FALSE || $this->streamWrapperManager->getViaScheme($scheme) instanceof LocalStream; - } - - /** - * Urlencode all the components of a remote filename. - * - * @param string $filename - * The filename of the file to be urlencoded. - * - * @return string - * The urlencoded filename. - */ - protected function urlencode($filename) { - // Only apply to a full URL - if ($this->configuration['urlencode'] && strpos($filename, '://')) { - $components = explode('/', $filename); - foreach ($components as $key => $component) { - $components[$key] = rawurlencode($component); - } - $filename = implode('/', $components); - // Actually, we don't want certain characters encoded - $filename = str_replace('%3A', ':', $filename); - $filename = str_replace('%3F', '?', $filename); - $filename = str_replace('%26', '&', $filename); - } - return $filename; - } - - /** - * {@inheritdoc} - */ protected function processStubRow(Row $row) { // We stub the uri value ourselves so we can create a real stub file for it. if (!$row->getDestinationProperty('uri')) { diff --git a/core/modules/file/src/Plugin/migrate/process/FileCopy.php b/core/modules/file/src/Plugin/migrate/process/FileCopy.php new file mode 100644 index 0000000..18f862f --- /dev/null +++ b/core/modules/file/src/Plugin/migrate/process/FileCopy.php @@ -0,0 +1,225 @@ +streamWrapperManager = $stream_wrappers; + $this->fileSystem = $file_system; + } + + /** + * {@inheritdoc} + */ + public static function create(ContainerInterface $container, array $configuration, $plugin_id, $plugin_definition) { + return new static( + $configuration, + $plugin_id, + $plugin_definition, + $container->get('stream_wrapper_manager'), + $container->get('file_system') + ); + } + + /** + * {@inheritdoc} + */ + public function transform($value, MigrateExecutableInterface $migrate_executable, Row $row, $destination_property) { + // If we're stubbing a file entity, return a uri of NULL so it will get + // stubbed by the general process. + if ($row->isStub()) { + return NULL; + } + list($source, $destination) = $value; + + // Ensure the source file exists, if it's a local URI or path. + if ($this->isLocalUri($source) && !file_exists($source)) { + throw new MigrateException("File '$source' does not exist."); + } + + // If the start and end file is exactly the same, there is nothing to do. + if ($this->isLocationUnchanged($source, $destination)) { + return $destination; + } + + $replace = $this->getOverwriteMode($row); + $success = $this->writeFile($source, $destination, $replace); + if (!$success) { + $dir = $this->getDirectory($destination); + if (file_prepare_directory($dir, FILE_CREATE_DIRECTORY)) { + $success = $this->writeFile($source, $destination, $replace); + } + else { + throw new MigrateException("Could not create directory '$dir'"); + } + } + + if ($success) { + return $destination; + } + else { + throw new MigrateException("File $source could not be copied to $destination."); + } + } + + /** + * Tries to move or copy a file. + * + * @param string $source + * The source path or URI. + * @param string $destination + * The destination path or URI. + * @param int $replace + * (optional) FILE_EXISTS_REPLACE (default) or FILE_EXISTS_RENAME. + * + * @return bool + * TRUE on success, FALSE on failure. + */ + protected function writeFile($source, $destination, $replace = FILE_EXISTS_REPLACE) { + if ($this->configuration['move']) { + return (boolean) file_unmanaged_move($source, $destination, $replace); + } + else { + $destination = file_destination($destination, $replace); + $source = $this->urlencode($source); + return @copy($source, $destination); + } + } + + /** + * Determines how to handle file conflicts. + * + * @param \Drupal\migrate\Row $row + * + * @return int + * Either FILE_EXISTS_REPLACE (default) or FILE_EXISTS_RENAME, depending + * on the current configuration. + */ + protected function getOverwriteMode(Row $row) { + if (!empty($this->configuration['rename'])) { + return FILE_EXISTS_RENAME; + } + return FILE_EXISTS_REPLACE; + } + + /** + * Returns the directory component of a URI or path. + * + * For URIs like public://foo.txt, the full physical path of public:// + * will be returned, since a scheme by itself will trip up certain file + * API functions (such as file_prepare_directory()). + * + * @param string $uri + * The URI or path. + * + * @return string|false + * The directory component of the path or URI, or FALSE if it could not + * be determined. + */ + protected function getDirectory($uri) { + $dir = $this->fileSystem->dirname($uri); + if (substr($dir, -3) == '://') { + return $this->fileSystem->realpath($dir); + } + else { + return $dir; + } + } + + /** + * Returns if the source and destination URIs represent identical paths. + * If either URI is a remote stream, will return FALSE. + * + * @param string $source + * The source URI. + * @param string $destination + * The destination URI. + * + * @return bool + * TRUE if the source and destination URIs refer to the same physical path, + * otherwise FALSE. + */ + protected function isLocationUnchanged($source, $destination) { + if ($this->isLocalUri($source) && $this->isLocalUri($destination)) { + return $this->fileSystem->realpath($source) === $this->fileSystem->realpath($destination); + } + else { + return FALSE; + } + } + + /** + * Returns if the given URI or path is considered local. + * + * A URI or path is considered local if it either has no scheme component, + * or the scheme is implemented by a stream wrapper which extends + * \Drupal\Core\StreamWrapper\LocalStream. + * + * @param string $uri + * The URI or path to test. + * + * @return bool + */ + protected function isLocalUri($uri) { + $scheme = $this->fileSystem->uriScheme($uri); + return $scheme === FALSE || $this->streamWrapperManager->getViaScheme($scheme) instanceof LocalStream; + } + + /** + * Urlencode all the components of a remote filename. + * + * @param string $filename + * The filename of the file to be urlencoded. + * + * @return string + * The urlencoded filename. + */ + protected function urlencode($filename) { + // Only apply to a full URL + if ($this->configuration['urlencode'] && strpos($filename, '://')) { + $components = explode('/', $filename); + foreach ($components as $key => $component) { + $components[$key] = rawurlencode($component); + } + $filename = implode('/', $components); + // Actually, we don't want certain characters encoded + $filename = str_replace('%3A', ':', $filename); + $filename = str_replace('%3F', '?', $filename); + $filename = str_replace('%26', '&', $filename); + } + return $filename; + } + +} diff --git a/core/modules/file/src/Plugin/migrate/process/Urlencode.php b/core/modules/file/src/Plugin/migrate/process/Urlencode.php new file mode 100644 index 0000000..7b922e4 --- /dev/null +++ b/core/modules/file/src/Plugin/migrate/process/Urlencode.php @@ -0,0 +1,37 @@ + $component) { + $components[$key] = rawurlencode($component); + } + $value = implode('/', $components); + // Actually, we don't want certain characters encoded + $value = str_replace('%3A', ':', $value); + $value = str_replace('%3F', '?', $value); + $value = str_replace('%26', '&', $value); + } + return $value; + } + +} diff --git a/core/modules/file/src/Plugin/migrate/source/d7/File.php b/core/modules/file/src/Plugin/migrate/source/d7/File.php index aa920bd..bc45a09 100644 --- a/core/modules/file/src/Plugin/migrate/source/d7/File.php +++ b/core/modules/file/src/Plugin/migrate/source/d7/File.php @@ -84,9 +84,7 @@ public function prepareRow(Row $row) { // At this point, $path could be an absolute path or a relative path, // depending on how the scheme's variable was set. So we need to shear out // the source_base_path in order to make them all relative. - // @todo https://www.drupal.org/node/2577871 Don't depend on destination - // configuration and figure out if this is even needed at all? - $path = str_replace($this->migration->getDestinationConfiguration()['source_base_path'], NULL, $path); + $path = str_replace($this->configuration['constants']['source_base_path'], NULL, $path); $row->setSourceProperty('filepath', $path); return parent::prepareRow($row); } diff --git a/core/modules/file/tests/src/Kernel/Migrate/EntityFileTest.php b/core/modules/file/tests/src/Kernel/Migrate/EntityFileTest.php index 90d968d..c467deb 100644 --- a/core/modules/file/tests/src/Kernel/Migrate/EntityFileTest.php +++ b/core/modules/file/tests/src/Kernel/Migrate/EntityFileTest.php @@ -252,7 +252,6 @@ public function __construct($configuration = []) { $configuration += array( 'source_base_path' => '', 'source_path_property' => 'filepath', - 'destination_path_property' => 'uri', 'move' => FALSE, 'urlencode' => FALSE, ); diff --git a/core/modules/migrate_drupal_ui/src/MigrateUpgradeRunBatch.php b/core/modules/migrate_drupal_ui/src/MigrateUpgradeRunBatch.php index 2916347..6ad8957 100644 --- a/core/modules/migrate_drupal_ui/src/MigrateUpgradeRunBatch.php +++ b/core/modules/migrate_drupal_ui/src/MigrateUpgradeRunBatch.php @@ -111,8 +111,9 @@ public static function run($initial_ids, $operation, $config, &$context) { if ($destination['plugin'] === 'entity:file') { // Make sure we have a single trailing slash. $source_base_path = rtrim($config['source_base_path'], '/') . '/'; - $destination['source_base_path'] = $source_base_path; - $migration->set('destination', $destination); + $source = $migration->getSourceConfiguration(); + $source['constants']['source_base_path'] = $source_base_path; + $migration->set('source', $source); } if ($migration) {