reverted: --- b/core/modules/file/src/Plugin/migrate/process/FileCopy.php +++ /dev/null @@ -1,207 +0,0 @@ - FALSE, - 'rename' => FALSE, - ); - parent::__construct($configuration, $plugin_id, $plugin_definition); - $this->streamWrapperManager = $stream_wrappers; - $this->fileSystem = $file_system; - } - - /** - * {@inheritdoc} - */ - public static function create(ContainerInterface $container, array $configuration, $plugin_id, $plugin_definition) { - return new static( - $configuration, - $plugin_id, - $plugin_definition, - $container->get('stream_wrapper_manager'), - $container->get('file_system') - ); - } - - /** - * {@inheritdoc} - */ - public function transform($value, MigrateExecutableInterface $migrate_executable, Row $row, $destination_property) { - // If we're stubbing a file entity, return a uri of NULL so it will get - // stubbed by the general process. - if ($row->isStub()) { - return NULL; - } - list($source, $destination) = $value; - - // Ensure the source file exists, if it's a local URI or path. - if ($this->isLocalUri($source) && !file_exists($source)) { - throw new MigrateException("File '$source' does not exist."); - } - - // If the start and end file is exactly the same, there is nothing to do. - if ($this->isLocationUnchanged($source, $destination)) { - return $destination; - } - - $replace = $this->getOverwriteMode(); - // We attempt the copy first to avoid calling file_prepare_directory() any - // more than absolutely necessary. - if ($this->writeFile($source, $destination, $replace)) { - return $destination; - } - $dir = $this->getDirectory($destination); - if (!file_prepare_directory($dir, FILE_CREATE_DIRECTORY)) { - throw new MigrateException("Could not create directory '$dir'"); - } - if ($this->writeFile($source, $destination, $replace)) { - return $destination; - } - throw new MigrateException("File $source could not be copied to $destination."); - } - - /** - * Tries to move or copy a file. - * - * @param string $source - * The source path or URI. - * @param string $destination - * The destination path or URI. - * @param int $replace - * (optional) FILE_EXISTS_REPLACE (default) or FILE_EXISTS_RENAME. - * - * @return bool - * TRUE on success, FALSE on failure. - */ - protected function writeFile($source, $destination, $replace = FILE_EXISTS_REPLACE) { - if ($this->configuration['move']) { - return (boolean) file_unmanaged_move($source, $destination, $replace); - } - $destination = file_destination($destination, $replace); - return @copy($source, $destination); - } - - /** - * Determines how to handle file conflicts. - * - * @return int - * Either FILE_EXISTS_REPLACE (default) or FILE_EXISTS_RENAME, depending - * on the current configuration. - */ - protected function getOverwriteMode() { - if (!empty($this->configuration['rename'])) { - return FILE_EXISTS_RENAME; - } - return FILE_EXISTS_REPLACE; - } - - /** - * Returns the directory component of a URI or path. - * - * For URIs like public://foo.txt, the full physical path of public:// - * will be returned, since a scheme by itself will trip up certain file - * API functions (such as file_prepare_directory()). - * - * @param string $uri - * The URI or path. - * - * @return string|false - * The directory component of the path or URI, or FALSE if it could not - * be determined. - */ - protected function getDirectory($uri) { - $dir = $this->fileSystem->dirname($uri); - if (substr($dir, -3) == '://') { - return $this->fileSystem->realpath($dir); - } - return $dir; - } - - /** - * Determines if the source and destination URIs represent identical paths. - * - * If either URI is a remote stream, will return FALSE. - * - * @param string $source - * The source URI. - * @param string $destination - * The destination URI. - * - * @return bool - * TRUE if the source and destination URIs refer to the same physical path, - * otherwise FALSE. - */ - protected function isLocationUnchanged($source, $destination) { - if ($this->isLocalUri($source) && $this->isLocalUri($destination)) { - return $this->fileSystem->realpath($source) === $this->fileSystem->realpath($destination); - } - return FALSE; - } - - /** - * Determines if the given URI or path is considered local. - * - * A URI or path is considered local if it either has no scheme component, - * or the scheme is implemented by a stream wrapper which extends - * \Drupal\Core\StreamWrapper\LocalStream. - * - * @param string $uri - * The URI or path to test. - * - * @return bool - */ - protected function isLocalUri($uri) { - $scheme = $this->fileSystem->uriScheme($uri); - return $scheme === FALSE || $this->streamWrapperManager->getViaScheme($scheme) instanceof LocalStream; - } - -} reverted: --- b/core/modules/file/src/Plugin/migrate/process/UrlEncode.php +++ /dev/null @@ -1,36 +0,0 @@ - 0) { - $components = explode('/', $value); - foreach ($components as $key => $component) { - // urlencode() would convert spaces to + signs. - $components[$key] = rawurlencode($component); - } - $value = implode('/', $components); - // Do not encode URL syntax characters. - $value = str_replace(['%3A', '%3F', '%26'], [':', '?', '&'], $value); - } - return $value; - } - -} diff -u b/core/modules/file/tests/src/Kernel/Migrate/CopyFileProcessPluginTest.php b/core/modules/file/tests/src/Kernel/Migrate/CopyFileProcessPluginTest.php --- b/core/modules/file/tests/src/Kernel/Migrate/CopyFileProcessPluginTest.php +++ b/core/modules/file/tests/src/Kernel/Migrate/CopyFileProcessPluginTest.php @@ -3,7 +3,7 @@ namespace Drupal\Tests\file\Kernel\Migrate; use Drupal\Core\StreamWrapper\StreamWrapperInterface; -use Drupal\file\Plugin\migrate\process\FileCopy; +use Drupal\migrate\Plugin\migrate\process\FileCopy; use Drupal\migrate\MigrateExecutableInterface; use Drupal\migrate\Row; use Drupal\KernelTests\KernelTestBase; diff -u b/core/modules/file/tests/src/Unit/Plugin/migrate/process/UrlEncodeTest.php b/core/modules/file/tests/src/Unit/Plugin/migrate/process/UrlEncodeTest.php --- b/core/modules/file/tests/src/Unit/Plugin/migrate/process/UrlEncodeTest.php +++ b/core/modules/file/tests/src/Unit/Plugin/migrate/process/UrlEncodeTest.php @@ -2,14 +2,14 @@ namespace Drupal\Tests\file\Unit\Plugin\migrate\process; -use Drupal\file\Plugin\migrate\process\UrlEncode; +use Drupal\migrate\Plugin\migrate\process\UrlEncode; use Drupal\migrate\MigrateExecutable; use Drupal\migrate\MigrateMessage; use Drupal\migrate\Row; use Drupal\Tests\migrate\Unit\MigrateTestCase; /** - * @coversDefaultClass \Drupal\file\Plugin\migrate\process\UrlEncode + * @coversDefaultClass \Drupal\migrate\Plugin\migrate\process\UrlEncode * @group file */ class UrlEncodeTest extends MigrateTestCase { only in patch2: unchanged: --- /dev/null +++ b/core/modules/migrate/src/Plugin/migrate/process/FileCopy.php @@ -0,0 +1,207 @@ + FALSE, + 'rename' => FALSE, + ); + parent::__construct($configuration, $plugin_id, $plugin_definition); + $this->streamWrapperManager = $stream_wrappers; + $this->fileSystem = $file_system; + } + + /** + * {@inheritdoc} + */ + public static function create(ContainerInterface $container, array $configuration, $plugin_id, $plugin_definition) { + return new static( + $configuration, + $plugin_id, + $plugin_definition, + $container->get('stream_wrapper_manager'), + $container->get('file_system') + ); + } + + /** + * {@inheritdoc} + */ + public function transform($value, MigrateExecutableInterface $migrate_executable, Row $row, $destination_property) { + // If we're stubbing a file entity, return a uri of NULL so it will get + // stubbed by the general process. + if ($row->isStub()) { + return NULL; + } + list($source, $destination) = $value; + + // Ensure the source file exists, if it's a local URI or path. + if ($this->isLocalUri($source) && !file_exists($source)) { + throw new MigrateSkipProcessException("File '$source' does not exist."); + } + + // If the start and end file is exactly the same, there is nothing to do. + if ($this->isLocationUnchanged($source, $destination)) { + return $destination; + } + + $replace = $this->getOverwriteMode(); + // We attempt the copy first to avoid calling file_prepare_directory() any + // more than absolutely necessary. + if ($this->writeFile($source, $destination, $replace)) { + return $destination; + } + $dir = $this->getDirectory($destination); + if (!file_prepare_directory($dir, FILE_CREATE_DIRECTORY)) { + throw new MigrateSkipProcessException("Could not create directory '$dir'"); + } + if ($this->writeFile($source, $destination, $replace)) { + return $destination; + } + throw new MigrateSkipProcessException("File $source could not be copied to $destination."); + } + + /** + * Tries to move or copy a file. + * + * @param string $source + * The source path or URI. + * @param string $destination + * The destination path or URI. + * @param int $replace + * (optional) FILE_EXISTS_REPLACE (default) or FILE_EXISTS_RENAME. + * + * @return bool + * TRUE on success, FALSE on failure. + */ + protected function writeFile($source, $destination, $replace = FILE_EXISTS_REPLACE) { + if ($this->configuration['move']) { + return (boolean) file_unmanaged_move($source, $destination, $replace); + } + $destination = file_destination($destination, $replace); + return @copy($source, $destination); + } + + /** + * Determines how to handle file conflicts. + * + * @return int + * Either FILE_EXISTS_REPLACE (default) or FILE_EXISTS_RENAME, depending + * on the current configuration. + */ + protected function getOverwriteMode() { + if (!empty($this->configuration['rename'])) { + return FILE_EXISTS_RENAME; + } + return FILE_EXISTS_REPLACE; + } + + /** + * Returns the directory component of a URI or path. + * + * For URIs like public://foo.txt, the full physical path of public:// + * will be returned, since a scheme by itself will trip up certain file + * API functions (such as file_prepare_directory()). + * + * @param string $uri + * The URI or path. + * + * @return string|false + * The directory component of the path or URI, or FALSE if it could not + * be determined. + */ + protected function getDirectory($uri) { + $dir = $this->fileSystem->dirname($uri); + if (substr($dir, -3) == '://') { + return $this->fileSystem->realpath($dir); + } + return $dir; + } + + /** + * Determines if the source and destination URIs represent identical paths. + * + * If either URI is a remote stream, will return FALSE. + * + * @param string $source + * The source URI. + * @param string $destination + * The destination URI. + * + * @return bool + * TRUE if the source and destination URIs refer to the same physical path, + * otherwise FALSE. + */ + protected function isLocationUnchanged($source, $destination) { + if ($this->isLocalUri($source) && $this->isLocalUri($destination)) { + return $this->fileSystem->realpath($source) === $this->fileSystem->realpath($destination); + } + return FALSE; + } + + /** + * Determines if the given URI or path is considered local. + * + * A URI or path is considered local if it either has no scheme component, + * or the scheme is implemented by a stream wrapper which extends + * \Drupal\Core\StreamWrapper\LocalStream. + * + * @param string $uri + * The URI or path to test. + * + * @return bool + */ + protected function isLocalUri($uri) { + $scheme = $this->fileSystem->uriScheme($uri); + return $scheme === FALSE || $this->streamWrapperManager->getViaScheme($scheme) instanceof LocalStream; + } + +} only in patch2: unchanged: --- /dev/null +++ b/core/modules/migrate/src/Plugin/migrate/process/UrlEncode.php @@ -0,0 +1,39 @@ + 0) { + $components = explode('/', $value); + foreach ($components as $key => $component) { + // urlencode() would convert spaces to + signs. + $components[$key] = rawurlencode($component); + } + $value = implode('/', $components); + // Do not encode URL syntax characters. + $value = str_replace(['%3A', '%3F', '%26'], [':', '?', '&'], $value); + } + return $value; + } + +} only in patch2: unchanged: --- /dev/null +++ b/core/modules/migrate/tests/src/Unit/process/CopyFileTest.php @@ -0,0 +1,148 @@ +container->get('stream_wrapper_manager')->registerWrapper('public', 'Drupal\Core\StreamWrapper\PublicStream', StreamWrapperInterface::NORMAL); + } + + /** + * Test successful imports/copies. + */ + public function testSuccessfulCopies() { + foreach ($this->localFileDataCopyProvider() as $data) { + list($source_path, $destination_path, $expected) = $data; + $this->doImport($source_path, $destination_path); + $message = $expected ? sprintf('File %s exists', $destination_path) : sprintf('File %s does not exist', $destination_path); + $this->assertSame($expected, is_file($destination_path), $message); + // Make sure we didn't accidentally do a move. + $message = $expected ? sprintf('File %s exists', $source_path) : sprintf('File %s does not exist', $source_path); + $this->assertSame($expected, is_file($source_path), $message); + } + } + + /** + * The data provider for testing the file destination. + * + * @return array + * An array of file permutations to test. + */ + protected function localFileDataCopyProvider() { + touch('/tmp/test-file.jpg'); + return [ + // Test a local to local copy. + [$this->root . '/core/modules/simpletest/files/image-test.jpg', 'public://file1.jpg', TRUE], + // Test a temporary file using an absolute path. + ['/tmp/test-file.jpg', 'temporary://test.jpg', TRUE], + // Test a temporary file using a relative path. + ['/tmp/test-file.jpg', 'temporary://core/modules/simpletest/files/test.jpg', TRUE], + ]; + } + + /** + * Test successful moves. + */ + public function testSuccessfulMoves() { + foreach ($this->localFileDataMoveProvider() as $data) { + list($source_path, $destination_path, $expected) = $data; + $this->doImport($source_path, $destination_path, ['move' => TRUE]); + $message = $expected ? sprintf('File %s exists', $destination_path) : sprintf('File %s does not exist', $destination_path); + $this->assertSame($expected, is_file($destination_path), $message); + $message = $expected ? sprintf('File %s does not exist', $source_path) : sprintf('File %s exists', $source_path); + $this->assertSame($expected, !is_file($source_path), $message); + } + } + + /** + * The data provider for testing the file destination. + * + * @return array + * An array of file permutations to test. + */ + protected function localFileDataMoveProvider() { + touch('/tmp/test-file.jpg'); + touch('/tmp/test-file2.jpg'); + $local_file = $this->root . '/sites/default/files/source_file.txt'; + touch($local_file); + return [ + // Test a local to local copy. + [$local_file, 'public://file1.jpg', TRUE], + // Test a temporary file using an absolute path. + ['/tmp/test-file.jpg', 'temporary://test.jpg', TRUE], + // Test a temporary file using a relative path. + ['/tmp/test-file2.jpg', 'temporary://core/modules/simpletest/files/test.jpg', TRUE], + ]; + } + + /** + * Test that non-existent files throw an exception. + * + * @expectedException \Drupal\migrate\MigrateException + * + * @expectedExceptionMessage File '/non/existent/file' does not exist + */ + public function testNonExistentSourceFile() { + $source = '/non/existent/file'; + $this->doImport($source, 'public://wontmatter.jpg'); + } + + /** + * Test the 'rename' overwrite mode. + */ + public function testRenameFile() { + $source = 'temporary://baz.txt'; + $destination = 'public://foo.txt'; + $expected_destination = 'public://foo_0.txt'; + touch($source); + touch($destination); + $this->doImport($source, $destination, ['rename' => TRUE]); + $this->assertFileExists($expected_destination); + } + + /** + * Do an import using the destination. + * + * @param string $source_path + * Source path to copy from. + * @param string $destination_path + * The destination path to copy to. + * @param array $configuration + * Process plugin configuration settings. + * + * @throws \Drupal\migrate\MigrateException + */ + protected function doImport($source_path, $destination_path, $configuration = []) { + $this->plugin = FileCopy::create($this->container, $configuration, 'file_copy', []); + $executable = $this->prophesize(MigrateExecutableInterface::class)->reveal(); + $row = new Row([], []); + + $result = $this->plugin->transform([$source_path, $destination_path], $executable, $row, 'foobaz'); + + // The plugin should either throw an exception or return the destination + // path. + $this->assertSame($result, $destination_path); + } + +} only in patch2: unchanged: --- /dev/null +++ b/core/modules/migrate/tests/src/Unit/process/UrlEncodeTest.php @@ -0,0 +1,59 @@ + 'test', + ]; + + /** + * Cover various encoding scenarios. + */ + public function testUrls() { + $values = [ + // A URL with no characters requiring encoding. + 'http://example.com/normal_url.html' => 'http://example.com/normal_url.html', + // The definitive use case - encoding spaces in URLs. + 'http://example.com/url with spaces.html' => 'http://example.com/url%20with%20spaces.html', + // Local filespecs should not be transformed, with or without spaces. + '/tmp/normal.txt' => '/tmp/normal.txt', + '/tmp/with spaces.txt' => '/tmp/with spaces.txt', + // Make sure URL characters (:, ?, &) are not encoded but others are. + 'https://example.com/?a=b@c&d=e+f%' => 'https://example.com/?a%3Db%40c&d%3De%2Bf%25', + ]; + + foreach ($values as $input => $output) { + $this->assertEquals($output, $this->doTransform($input)); + } + } + + /** + * Perform the urlencode process plugin over the given value. + * + * @param string $value + * Url to be encoded. + * + * @return string + * Encoded url. + */ + protected function doTransform($value) { + $executable = new MigrateExecutable($this->getMigration(), new MigrateMessage()); + $row = new Row([], []); + + return (new UrlEncode([], 'urlencode', [])) + ->transform($value, $executable, $row, 'foobaz'); + } + +}