diff -u b/core/modules/file/migration_templates/d7_file.yml b/core/modules/file/migration_templates/d7_file.yml --- b/core/modules/file/migration_templates/d7_file.yml +++ b/core/modules/file/migration_templates/d7_file.yml @@ -23,7 +23,6 @@ source: - '@source_full_path' - uri - urlencode: true filemime: filemime # filesize is dynamically computed when file entities are saved, so there is # no point in migrating it. diff -u b/core/modules/file/src/Plugin/migrate/process/FileCopy.php b/core/modules/file/src/Plugin/migrate/process/FileCopy.php --- b/core/modules/file/src/Plugin/migrate/process/FileCopy.php +++ b/core/modules/file/src/Plugin/migrate/process/FileCopy.php @@ -35,6 +35,10 @@ * {@inheritdoc} */ public function __construct(array $configuration, $plugin_id, array $plugin_definition, StreamWrapperManagerInterface $stream_wrappers, FileSystemInterface $file_system) { + $configuration += array( + 'move' => FALSE, + 'rename' => FALSE, + ); parent::__construct($configuration, $plugin_id, $plugin_definition); $this->streamWrapperManager = $stream_wrappers; $this->fileSystem = $file_system; @@ -113,7 +117,6 @@ } else { $destination = file_destination($destination, $replace); - $source = $this->urlencode($source); return @copy($source, $destination); } } @@ -199,27 +202,2 @@ - /** - * Urlencode all the components of a remote filename. - * - * @param string $filename - * The filename of the file to be urlencoded. - * - * @return string - * The urlencoded filename. - */ - protected function urlencode($filename) { - // Only apply to a full URL - if ($this->configuration['urlencode'] && strpos($filename, '://')) { - $components = explode('/', $filename); - foreach ($components as $key => $component) { - $components[$key] = rawurlencode($component); - } - $filename = implode('/', $components); - // Actually, we don't want certain characters encoded - $filename = str_replace('%3A', ':', $filename); - $filename = str_replace('%3F', '?', $filename); - $filename = str_replace('%26', '&', $filename); - } - return $filename; - } - } diff -u b/core/modules/file/tests/src/Kernel/Migrate/EntityFileTest.php /dev/null --- b/core/modules/file/tests/src/Kernel/Migrate/EntityFileTest.php +++ /dev/null @@ -1,277 +0,0 @@ -get('stream_wrapper_manager')->registerWrapper('public', 'Drupal\Core\StreamWrapper\PublicStream', StreamWrapperInterface::NORMAL); - $this->destination = new TestEntityFile([]); - $this->installEntitySchema('file'); - - file_put_contents('/tmp/test-file.jpg', ''); - } - - /** - * Test successful imports/copies. - */ - public function testSuccessfulCopies() { - foreach ($this->localFileDataProvider() as $data) { - list($row_values, $destination_path, $expected, $source_base_path) = $data; - - $this->doImport($row_values, $destination_path, $source_base_path); - $message = $expected ? sprintf('File %s exists', $destination_path) : sprintf('File %s does not exist', $destination_path); - $this->assertIdentical($expected, is_file($destination_path), $message); - } - } - - /** - * The data provider for testing the file destination. - * - * @return array - * An array of file permutations to test. - */ - protected function localFileDataProvider() { - return [ - // Test a local to local copy. - [['filepath' => 'core/modules/simpletest/files/image-test.jpg'], 'public://file1.jpg', TRUE, $this->root . '/'], - // Test a temporary file using an absolute path. - [['filepath' => '/tmp/test-file.jpg'], 'temporary://test.jpg', TRUE, ''], - // Test a temporary file using a relative path. - [['filepath' => 'test-file.jpg'], 'temporary://core/modules/simpletest/files/test.jpg', TRUE, '/tmp/'], - // Test a remote path to local. - [['filepath' => 'core/modules/simpletest/files/image-test.jpg'], 'public://remote-file.jpg', TRUE, $this->root . '/'], - // Test a remote path to local inside a folder that doesn't exist. - [['filepath' => 'core/modules/simpletest/files/image-test.jpg'], 'public://folder/remote-file.jpg', TRUE, $this->root . '/'], - ]; - } - - /** - * Test that non-existent files throw an exception. - */ - public function testNonExistentSourceFile() { - $destination = '/non/existent/file'; - try { - // If this test passes, doImport() will raise a MigrateException and - // we'll never reach fail(). - $this->doImport(['filepath' => $destination], 'public://wontmatter.jpg'); - $this->fail('Expected Drupal\migrate\MigrateException when importing ' . $destination); - } - catch (MigrateException $e) { - $this->assertIdentical($e->getMessage(), "File '$destination' does not exist."); - } - } - - /** - * Tests various invocations of the writeFile() method. - */ - public function testWriteFile() { - $plugin = $this->destination; - $method = new \ReflectionMethod($plugin, 'writeFile'); - $method->setAccessible(TRUE); - - touch('temporary://baz.txt'); - - // Moving an actual file should return TRUE. - $plugin->configuration['move'] = TRUE; - $this->assertTrue($method->invoke($plugin, 'temporary://baz.txt', 'public://foo.txt')); - - // Trying to move a non-existent file should return FALSE. - $this->assertFalse($method->invoke($plugin, 'temporary://invalid.txt', 'public://invalid.txt')); - - // Copying over a file that already exists should replace the existing file. - $plugin->configuration['move'] = FALSE; - touch('temporary://baz.txt'); - $this->assertTrue($method->invoke($plugin, 'temporary://baz.txt', 'public://foo.txt')); - // Copying over a file that already exists should rename the resulting file - // if FILE_EXISTS_RENAME is specified. - $method->invoke($plugin, 'temporary://baz.txt', 'public://foo.txt', FILE_EXISTS_RENAME); - $this->assertTrue(file_exists('public://foo_0.txt')); - - // Trying to copy a non-existent file should return FALSE. - $this->assertFalse($method->invoke($plugin, 'temporary://invalid.txt', 'public://invalid.txt')); - } - - /** - * Tests various invocations of the getOverwriteMode() method. - */ - public function testGetOverwriteMode() { - $plugin = $this->destination; - $method = new \ReflectionMethod($plugin, 'getOverwriteMode'); - $method->setAccessible(TRUE); - - $row = new Row([], []); - // If the plugin is not configured to rename the destination file, we should - // always get FILE_EXISTS_REPLACE. - $this->assertIdentical(FILE_EXISTS_REPLACE, $method->invoke($plugin, $row)); - - // When the plugin IS configured to rename the destination file, it should - // return FILE_EXISTS_RENAME if the destination entity already exists, - // and FILE_EXISTS_REPLACE otherwise. - $plugin->configuration['rename'] = TRUE; - $plugin->storage = \Drupal::entityManager()->getStorage('file'); - /** @var \Drupal\file\FileInterface $file */ - $file = $plugin->storage->create(); - touch('public://foo.txt'); - $file->setFileUri('public://foo.txt'); - $file->save(); - $row->setDestinationProperty($plugin->storage->getEntityType()->getKey('id'), $file->id()); - $this->assertIdentical(FILE_EXISTS_RENAME, $method->invoke($plugin, $row)); - unlink('public://foo.txt'); - } - - /** - * Tests various invocations of the getDirectory() method. - */ - public function testGetDirectory() { - $plugin = $this->destination; - $method = new \ReflectionMethod($plugin, 'getDirectory'); - $method->setAccessible(TRUE); - - $this->assertSame('public://foo', $method->invoke($plugin, 'public://foo/baz.txt')); - $this->assertSame('/path/to', $method->invoke($plugin, '/path/to/foo.txt')); - // A directory like public:// (no path) needs to resolve to a physical path. - $fs = \Drupal::getContainer()->get('file_system'); - $this->assertSame($fs->realpath('public://'), $method->invoke($plugin, 'public://foo.txt')); - } - - /** - * Tests various invocations of the isLocationUnchanged() method. - */ - public function testIsLocationUnchanged() { - $plugin = $this->destination; - $method = new \ReflectionMethod($plugin, 'isLocationUnchanged'); - $method->setAccessible(TRUE); - - $temporary_file = '/tmp/foo.txt'; - touch($temporary_file); - $this->assertTrue($method->invoke($plugin, $temporary_file, 'temporary://foo.txt')); - unlink($temporary_file); - } - - /** - * Tests various invocations of the isLocalUri() method. - */ - public function testIsLocalUri() { - $plugin = $this->destination; - $method = new \ReflectionMethod($plugin, 'isLocalUri'); - $method->setAccessible(TRUE); - - $this->assertTrue($method->invoke($plugin, 'public://foo.txt')); - $this->assertTrue($method->invoke($plugin, 'public://path/to/foo.txt')); - $this->assertTrue($method->invoke($plugin, 'temporary://foo.txt')); - $this->assertTrue($method->invoke($plugin, 'temporary://path/to/foo.txt')); - $this->assertTrue($method->invoke($plugin, 'foo.txt')); - $this->assertTrue($method->invoke($plugin, '/path/to/files/foo.txt')); - $this->assertTrue($method->invoke($plugin, 'relative/path/to/foo.txt')); - $this->assertFalse($method->invoke($plugin, 'http://www.example.com/foo.txt')); - } - - /** - * Do an import using the destination. - * - * @param array $row_values - * An array of row values. - * @param string $destination_path - * The destination path to copy to. - * @param string $source_base_path - * The source base path. - * @return array - * An array of saved entities ids. - * - * @throws \Drupal\migrate\MigrateException - */ - protected function doImport($row_values, $destination_path, $source_base_path = '') { - $row = new Row($row_values, []); - $row->setDestinationProperty('uri', $destination_path); - $this->destination->configuration['source_base_path'] = $source_base_path; - - // Importing asserts there are no errors, then we just check the file has - // been copied into place. - return $this->destination->import($row, array()); - } - -} - -class TestEntityFile extends EntityFile { - - /** - * This is needed to be passed to $this->save(). - * - * @var \Drupal\Core\Entity\ContentEntityInterface - */ - public $mockEntity; - - /** - * Make this public for easy writing during tests. - * - * @var array - */ - public $configuration; - - /** - * @var \Drupal\Core\Entity\EntityStorageInterface - */ - public $storage; - - public function __construct($configuration = []) { - $configuration += array( - 'source_base_path' => '', - 'source_path_property' => 'filepath', - 'move' => FALSE, - 'urlencode' => FALSE, - ); - $this->configuration = $configuration; - // We need a mock entity to be passed to save to prevent strict exceptions. - $this->mockEntity = EntityTest::create(); - $this->streamWrapperManager = \Drupal::getContainer()->get('stream_wrapper_manager'); - $this->fileSystem = \Drupal::getContainer()->get('file_system'); - } - - /** - * {@inheritdoc} - */ - protected function getEntity(Row $row, array $old_destination_id_values) { - return $this->mockEntity; - } - - /** - * {@inheritdoc} - */ - protected function save(ContentEntityInterface $entity, array $old_destination_id_values = array()) {} - -} only in patch2: unchanged: --- /dev/null +++ b/core/modules/file/tests/src/Kernel/Migrate/CopyFileProcessPluginTest.php @@ -0,0 +1,165 @@ +get('stream_wrapper_manager')->registerWrapper('public', 'Drupal\Core\StreamWrapper\PublicStream', StreamWrapperInterface::NORMAL); + } + + /** + * Test successful imports/copies. + */ + public function testSuccessfulCopies() { + foreach ($this->localFileDataCopyProvider() as $data) { + list($source_path, $destination_path, $expected) = $data; + + $this->doImport($source_path, $destination_path); + $message = $expected ? sprintf('File %s exists', $destination_path) : sprintf('File %s does not exist', $destination_path); + $this->assertSame($expected, is_file($destination_path), $message); + // Make sure we didn't accidentally do a move. + $message = $expected ? sprintf('File %s exists', $source_path) : sprintf('File %s does not exist', $source_path); + $this->assertSame($expected, is_file($source_path), $message); + } + } + + /** + * The data provider for testing the file destination. + * + * @return array + * An array of file permutations to test. + */ + protected function localFileDataCopyProvider() { + touch('/tmp/test-file.jpg'); + return [ + // Test a local to local copy. + [$this->root . '/core/modules/simpletest/files/image-test.jpg', 'public://file1.jpg', TRUE], + // Test a temporary file using an absolute path. + ['/tmp/test-file.jpg', 'temporary://test.jpg', TRUE], + // Test a temporary file using a relative path. + ['/tmp/test-file.jpg', 'temporary://core/modules/simpletest/files/test.jpg', TRUE], + ]; + } + + /** + * Test successful moves. + */ + public function testSuccessfulMoves() { + foreach ($this->localFileDataMoveProvider() as $data) { + list($source_path, $destination_path, $expected) = $data; + + $this->doImport($source_path, $destination_path, ['move' => TRUE]); + $message = $expected ? sprintf('File %s exists', $destination_path) : sprintf('File %s does not exist', $destination_path); + $this->assertSame($expected, is_file($destination_path), $message); + $message = $expected ? sprintf('File %s does not exist', $source_path) : sprintf('File %s exists', $source_path); + $this->assertSame($expected, !is_file($source_path), $message); + } + } + + /** + * The data provider for testing the file destination. + * + * @return array + * An array of file permutations to test. + */ + protected function localFileDataMoveProvider() { + touch('/tmp/test-file.jpg'); + touch('/tmp/test-file2.jpg'); + $local_file = $this->root . '/sites/default/files/source_file.txt'; + touch($local_file); + return [ + // Test a local to local copy. + [$local_file, 'public://file1.jpg', TRUE], + // Test a temporary file using an absolute path. + ['/tmp/test-file.jpg', 'temporary://test.jpg', TRUE], + // Test a temporary file using a relative path. + ['/tmp/test-file2.jpg', 'temporary://core/modules/simpletest/files/test.jpg', TRUE], + ]; + } + + /** + * Test that non-existent files throw an exception. + */ + public function testNonExistentSourceFile() { + $source = '/non/existent/file'; + try { + // If this test passes, doImport() will raise a MigrateException and + // we'll never reach fail(). + $this->doImport($source, 'public://wontmatter.jpg'); + $this->fail('Expected Drupal\migrate\MigrateException when importing ' . $source); + } + catch (MigrateException $e) { + $this->assertIdentical($e->getMessage(), "File '$source' does not exist."); + } + } + + /** + * Test the 'rename' overwrite mode. + */ + public function testRenameFile() { + $source = 'temporary://baz.txt'; + $destination = 'public://foo.txt'; + $expected_destination = 'public://foo_0.txt'; + touch($source); + touch($destination); + $this->doImport($source, $destination, ['rename' => TRUE]); + $this->assertTrue(is_file($expected_destination)); + } + + /** + * Do an import using the destination. + * + * @param string $source_path + * Source path to copy from. + * @param string $destination_path + * The destination path to copy to. + * @param array $configuration + * Process plugin configuration settings. + * + * @return array + * An array of saved entities ids. + * + * @throws \Drupal\migrate\MigrateException + */ + protected function doImport($source_path, $destination_path, $configuration = []) { + $this->plugin = FileCopy::create(\Drupal::getContainer(), $configuration, 'file_copy', []); + $executable = $this->prophesize(MigrateExecutableInterface::class)->reveal(); + $row = new Row([], []); + + $result = $this->plugin->transform([$source_path, $destination_path], $executable, $row, 'foobaz'); + + // The plugin should either throw an exception or return the destination + // path. + $this->assertSame($result, $destination_path); + } + +} only in patch2: unchanged: --- a/core/modules/file/tests/src/Kernel/Migrate/d7/MigrateFileTest.php +++ b/core/modules/file/tests/src/Kernel/Migrate/d7/MigrateFileTest.php @@ -33,17 +33,11 @@ protected function setUp() { /** @var \Drupal\migrate\Plugin\Migration $migration */ $migration = $this->getMigration('d7_file'); - // Set the destination plugin's source_base_path configuration value, which + // Set the source plugin's source_base_path configuration value, which // would normally be set by the user running the migration. - $migration->set('destination', [ - 'plugin' => 'entity:file', - // Note that source_base_path must include a trailing slash because it's - // prepended directly to the value of the source path property. - 'source_base_path' => $fs->realpath('public://') . '/', - // This is set in the migration's YAML file, but we need to repeat it - // here because all the destination configuration must be set at once. - 'source_path_property' => 'filepath', - ]); + $source = $migration->getSourceConfiguration(); + $source['constants']['source_base_path'] = $fs->realpath('public://') . '/'; + $migration->set('source', $source); $this->executeMigration($migration); } only in patch2: unchanged: --- /dev/null +++ b/core/modules/file/tests/src/Unit/Plugin/migrate/process/FileCopyTest.php @@ -0,0 +1,26 @@ +prophesize(StreamWrapperManagerInterface::class); + $file_system = $this->prophesize(FileSystemInterface::class); + $this->plugin = new FileCopy([], 'file_copy', [], $stream_wrapper_manager->reveal(), $file_system->reveal()); + } + +} only in patch2: unchanged: --- /dev/null +++ b/core/modules/file/tests/src/Unit/Plugin/migrate/process/UrlencodeTest.php @@ -0,0 +1,55 @@ + 'http://example.com/normal_url.html', + // The definitive use case - encoding spaces in URLs. + 'http://example.com/url with spaces.html' => 'http://example.com/url%20with%20spaces.html', + // Local filespecs should not be transformed, with or without spaces. + '/tmp/normal.txt' => '/tmp/normal.txt', + '/tmp/with spaces.txt' => '/tmp/with spaces.txt', + // Make sure URL characters (:, ?, &) are not encoded but others are. + 'https://example.com/?a=b@c&d=e+f%' => 'https://example.com/?a%3Db%40c&d%3De%2Bf%25', + ]; + + foreach ($values as $input => $output) { + $this->assertEquals($output, $this->doTransform($input)); + } + } + + /** + * Perform the urlencode process plugin over the given value. + * + * @param $value + * Url to be encoded. + * + * @return string + * Encoded url. + */ + protected function doTransform($value) { + $executable = new MigrateExecutable($this->getMigration(), new MigrateMessage()); + $row = new Row([], []); + + return (new Urlencode([], 'urlencode', [])) + ->transform($value, $executable, $row, 'foobaz'); + } + +}