diff --git a/core/lib/Drupal/Core/Database/Driver/mysql/Connection.php b/core/lib/Drupal/Core/Database/Driver/mysql/Connection.php index 56e4f2d..ed97514 100644 --- a/core/lib/Drupal/Core/Database/Driver/mysql/Connection.php +++ b/core/lib/Drupal/Core/Database/Driver/mysql/Connection.php @@ -64,7 +64,7 @@ public static function open(array &$connection_options = array()) { // Character set is added to dsn to ensure PDO uses the proper character // set when escaping. This has security implications. See // https://www.drupal.org/node/1201452 for further discussion. - $dsn .= ';charset=utf8'; + $dsn .= ';charset=utf8mb4'; if (!empty($connection_options['database'])) { $dsn .= ';dbname=' . $connection_options['database']; } @@ -92,13 +92,13 @@ public static function open(array &$connection_options = array()) { $pdo = new \PDO($dsn, $connection_options['username'], $connection_options['password'], $connection_options['pdo']); // Force MySQL to use the UTF-8 character set. Also set the collation, if a - // certain one has been set; otherwise, MySQL defaults to 'utf8_general_ci' - // for UTF-8. + // certain one has been set; otherwise, MySQL defaults to + // 'utf8mb4_general_ci' for utf8mb4. if (!empty($connection_options['collation'])) { - $pdo->exec('SET NAMES utf8 COLLATE ' . $connection_options['collation']); + $pdo->exec('SET NAMES utf8mb4 COLLATE ' . $connection_options['collation']); } else { - $pdo->exec('SET NAMES utf8'); + $pdo->exec('SET NAMES utf8mb4'); } // Set MySQL init_commands if not already defined. Default Drupal's MySQL diff --git a/core/lib/Drupal/Core/Database/Driver/mysql/Schema.php b/core/lib/Drupal/Core/Database/Driver/mysql/Schema.php index 8237c79..6326bcf 100644 --- a/core/lib/Drupal/Core/Database/Driver/mysql/Schema.php +++ b/core/lib/Drupal/Core/Database/Driver/mysql/Schema.php @@ -87,7 +87,7 @@ protected function createTableSql($name, $table) { // Provide defaults if needed. $table += array( 'mysql_engine' => 'InnoDB', - 'mysql_character_set' => 'utf8', + 'mysql_character_set' => 'utf8mb4', ); $sql = "CREATE TABLE {" . $name . "} (\n"; @@ -108,8 +108,8 @@ protected function createTableSql($name, $table) { $sql .= 'ENGINE = ' . $table['mysql_engine'] . ' DEFAULT CHARACTER SET ' . $table['mysql_character_set']; // By default, MySQL uses the default collation for new tables, which is - // 'utf8_general_ci' for utf8. If an alternate collation has been set, it - // needs to be explicitly specified. + // 'utf8mb4_general_ci' for utf8mb4. If an alternate collation has been + // set, it needs to be explicitly specified. // @see DatabaseConnection_mysql if (!empty($info['collation'])) { $sql .= ' COLLATE ' . $info['collation']; diff --git a/core/modules/aggregator/src/FeedStorageSchema.php b/core/modules/aggregator/src/FeedStorageSchema.php index d251ed5..4d51bdb 100644 --- a/core/modules/aggregator/src/FeedStorageSchema.php +++ b/core/modules/aggregator/src/FeedStorageSchema.php @@ -33,7 +33,7 @@ protected function getSharedTableFieldSchema(FieldStorageDefinitionInterface $st break; case 'title': - $this->addSharedTableFieldUniqueKey($storage_definition, $schema); + $this->addSharedTableFieldIndex($storage_definition, $schema, TRUE); break; } } diff --git a/core/modules/block_content/src/BlockContentStorageSchema.php b/core/modules/block_content/src/BlockContentStorageSchema.php deleted file mode 100644 index c1674ae..0000000 --- a/core/modules/block_content/src/BlockContentStorageSchema.php +++ /dev/null @@ -1,52 +0,0 @@ - array('info', 'langcode'), - ); - - return $schema; - } - - /** - * {@inheritdoc} - */ - protected function getSharedTableFieldSchema(FieldStorageDefinitionInterface $storage_definition, $table_name, array $column_mapping) { - $schema = parent::getSharedTableFieldSchema($storage_definition, $table_name, $column_mapping); - $field_name = $storage_definition->getName(); - - if ($table_name == 'block_content_field_data') { - switch ($field_name) { - case 'info': - // Improves the performance of the block_content__info index defined - // in getEntitySchema(). - $schema['fields'][$field_name]['not null'] = TRUE; - break; - } - } - - return $schema; - } - -} diff --git a/core/modules/block_content/src/Entity/BlockContent.php b/core/modules/block_content/src/Entity/BlockContent.php index 046640c..76fe365 100644 --- a/core/modules/block_content/src/Entity/BlockContent.php +++ b/core/modules/block_content/src/Entity/BlockContent.php @@ -22,7 +22,7 @@ * bundle_label = @Translation("Custom block type"), * handlers = { * "storage" = "Drupal\Core\Entity\Sql\SqlContentEntityStorage", - * "storage_schema" = "Drupal\block_content\BlockContentStorageSchema", + * "storage_schema" = "Drupal\Core\Entity\Sql\SqlContentEntityStorageSchema", * "access" = "Drupal\block_content\BlockContentAccessControlHandler", * "list_builder" = "Drupal\block_content\BlockContentListBuilder", * "view_builder" = "Drupal\block_content\BlockContentViewBuilder", diff --git a/core/modules/file/src/Entity/File.php b/core/modules/file/src/Entity/File.php index ff38ab3..1265d31 100644 --- a/core/modules/file/src/Entity/File.php +++ b/core/modules/file/src/Entity/File.php @@ -7,6 +7,7 @@ namespace Drupal\file\Entity; +use Drupal\Component\Utility\Crypt; use Drupal\Core\Entity\ContentEntityBase; use Drupal\Core\Entity\EntityStorageInterface; use Drupal\Core\Entity\EntityTypeInterface; @@ -194,7 +195,10 @@ public static function preCreate(EntityStorageInterface $storage, array &$values public function preSave(EntityStorageInterface $storage) { parent::preSave($storage); - $this->setSize(filesize($this->getFileUri())); + $uri = $this->getFileUri(); + $this->setSize(filesize($uri)); + // Save the hash of the URI so we can enforce uniqueness. + $this->get('uri_hash')->value = Crypt::hashBase64($uri); } /** @@ -252,6 +256,15 @@ public static function baseFieldDefinitions(EntityTypeInterface $entity_type) { ->setSetting('max_length', 255) ->setSetting('case_sensitive', TRUE); + $fields['uri_hash'] = BaseFieldDefinition::create('string') + ->setLabel(t('URI Hash')) + ->setDescription(t('The hashed URI, stored to enforce URI uniqueness.')) + ->setSetting('is_ascii', TRUE) + // As this is a base64-encoded SHA-256 hash, we can limit the length to + // 100 characters. + ->setSetting('max_length', 100) + ->setSetting('case_sensitive', TRUE); + $fields['filemime'] = BaseFieldDefinition::create('string') ->setLabel(t('File MIME type')) ->setSetting('is_ascii', TRUE) diff --git a/core/modules/file/src/FileStorageSchema.php b/core/modules/file/src/FileStorageSchema.php index f253020..e21850b 100644 --- a/core/modules/file/src/FileStorageSchema.php +++ b/core/modules/file/src/FileStorageSchema.php @@ -30,7 +30,10 @@ protected function getSharedTableFieldSchema(FieldStorageDefinitionInterface $st break; case 'uri': - $this->addSharedTableFieldUniqueKey($storage_definition, $schema, TRUE); + $this->addSharedTableFieldIndex($storage_definition, $schema, TRUE); + break; + case 'uri_hash': + $this->addSharedTableFieldUniqueKey($storage_definition, $schema); break; } } diff --git a/core/modules/file/src/Tests/SaveTest.php b/core/modules/file/src/Tests/SaveTest.php index cc12515..95d6806 100644 --- a/core/modules/file/src/Tests/SaveTest.php +++ b/core/modules/file/src/Tests/SaveTest.php @@ -8,6 +8,9 @@ namespace Drupal\file\Tests; use Drupal\file\Entity\File; +use Drupal\Core\Entity\EntityStorageException; +use Drupal\Core\Entity\EntityStorageInterface; +use Drupal\Core\Entity\Sql\SqlEntityStorageInterface; /** * File saving tests. @@ -59,7 +62,7 @@ function testFileSave() { // Try to insert a second file with the same name apart from case insensitivity // to ensure the 'uri' index allows for filenames with different cases. - $uppercase_file = File::create(array( + $uppercase_values = array( 'uid' => 1, 'filename' => 'DRUPLICON.txt', 'uri' => 'public://DRUPLICON.txt', @@ -67,10 +70,28 @@ function testFileSave() { 'created' => 1, 'changed' => 1, 'status' => FILE_STATUS_PERMANENT, - )); + ); + $uppercase_file = File::create($uppercase_values); file_put_contents($uppercase_file->getFileUri(), 'hello world'); $uppercase_file->save(); + $storage = \Drupal::entityManager()->getStorage('file'); + $this->assertTrue($storage instanceof EntityStorageInterface, 'Storage object implements EntityStorageInterface.'); + if ($storage instanceof SqlEntityStorageInterface) { + // Ensure the database URI uniqueness constraint is triggered. + $uppercase_file_duplicate = File::create($uppercase_values); + file_put_contents($uppercase_file_duplicate->getFileUri(), 'hello world'); + $constraint_triggered = NULL; + try { + $uppercase_file_duplicate->save(); + } + catch (EntityStorageException $e) { + // SQL error codes for unique constraints begin with 23. + $constraint_triggered = (0 === strpos($e->getCode(), '23')); + } + $this->assertTrue($constraint_triggered, 'SQL uniqueness constraint is triggered'); + } + // Ensure that file URI entity queries are case sensitive. $fids = \Drupal::entityQuery('file') ->condition('uri', $uppercase_file->getFileUri()) diff --git a/core/modules/migrate/src/Plugin/migrate/id_map/Sql.php b/core/modules/migrate/src/Plugin/migrate/id_map/Sql.php index 435907b..b30679c 100644 --- a/core/modules/migrate/src/Plugin/migrate/id_map/Sql.php +++ b/core/modules/migrate/src/Plugin/migrate/id_map/Sql.php @@ -261,6 +261,12 @@ protected function ensureTables() { foreach ($this->migration->getSourcePlugin()->getIds() as $id_definition) { $mapkey = 'sourceid' . $count++; $source_id_schema[$mapkey] = $this->getFieldSchema($id_definition); + + // With InnoDB, utf8mb4-based primary keys can't be over 191 characters. + // Use ASCII-based primary keys instead. + if (isset($source_id_schema[$mapkey]['type']) && $source_id_schema[$mapkey]['type'] == 'varchar') { + $source_id_schema[$mapkey]['type'] = 'varchar_ascii'; + } $pks[] = $mapkey; } diff --git a/core/modules/migrate_drupal/src/Tests/Table/d6/System.php b/core/modules/migrate_drupal/src/Tests/Table/d6/System.php index ae42df9..c185c35 100644 --- a/core/modules/migrate_drupal/src/Tests/Table/d6/System.php +++ b/core/modules/migrate_drupal/src/Tests/Table/d6/System.php @@ -26,7 +26,7 @@ public function load() { ), 'fields' => array( 'filename' => array( - 'type' => 'varchar', + 'type' => 'varchar_ascii', 'not null' => TRUE, 'length' => '255', 'default' => '', diff --git a/core/modules/migrate_drupal/src/Tests/d6/MigrateFileTest.php b/core/modules/migrate_drupal/src/Tests/d6/MigrateFileTest.php index 91560a4..89834c6 100644 --- a/core/modules/migrate_drupal/src/Tests/d6/MigrateFileTest.php +++ b/core/modules/migrate_drupal/src/Tests/d6/MigrateFileTest.php @@ -8,6 +8,7 @@ namespace Drupal\migrate_drupal\Tests\d6; use Drupal\Component\Utility\Random; +use Drupal\Component\Utility\Crypt; use Drupal\migrate\MigrateExecutable; use Drupal\migrate\Tests\MigrateDumpAlterInterface; use Drupal\migrate_drupal\Tests\d6\MigrateDrupal6TestBase; @@ -63,6 +64,7 @@ public function testFiles() { $this->assertIdentical('Image1.png', $file->getFilename()); $this->assertIdentical('39325', $file->getSize()); $this->assertIdentical('public://image-1.png', $file->getFileUri()); + $this->assertIdentical(Crypt::hashBase64('public://image-1.png'), $file->get('uri_hash')->value); $this->assertIdentical('image/png', $file->getMimeType()); // It is pointless to run the second half from MigrateDrupal6Test. if (empty($this->standalone)) { diff --git a/core/modules/node/src/Tests/NodeViewTest.php b/core/modules/node/src/Tests/NodeViewTest.php index 2281dd8..c60d44a 100644 --- a/core/modules/node/src/Tests/NodeViewTest.php +++ b/core/modules/node/src/Tests/NodeViewTest.php @@ -33,4 +33,16 @@ public function testHtmlHeadLinks() { $this->assertEqual($result[0]['href'], $node->url()); } + /** + * Tests that we store and retrieve multi-byte UTF-8 characters correctly. + */ + public function testMultiByteUtf8() { + $title = '🐝'; + $this->assertTrue(mb_strlen($title, 'utf-8') < strlen($title), 'Title has multi-byte characters.'); + $node = $this->drupalCreateNode(array('title' => $title)); + $this->drupalGet($node->urlInfo()); + $result = $this->xpath('//span[contains(@class, "field-name-title")]'); + $this->assertEqual((string) $result[0], $title, 'The passed title was returned.'); + } + } diff --git a/core/modules/system/src/Tests/Database/RegressionTest.php b/core/modules/system/src/Tests/Database/RegressionTest.php index 81f962d..6f87d2a 100644 --- a/core/modules/system/src/Tests/Database/RegressionTest.php +++ b/core/modules/system/src/Tests/Database/RegressionTest.php @@ -26,16 +26,16 @@ class RegressionTest extends DatabaseTestBase { */ function testRegression_310447() { // That's a 255 character UTF-8 string. - $name = str_repeat("é", 255); + $job = str_repeat("é", 255); db_insert('test') ->fields(array( - 'name' => $name, + 'name' => $this->randomMachineName(), 'age' => 20, - 'job' => 'Dancer', + 'job' => $job, ))->execute(); - $from_database = db_query('SELECT name FROM {test} WHERE name = :name', array(':name' => $name))->fetchField(); - $this->assertIdentical($name, $from_database, 'The database handles UTF-8 characters cleanly.'); + $from_database = db_query('SELECT job FROM {test} WHERE job = :job', array(':job' => $job))->fetchField(); + $this->assertIdentical($job, $from_database, 'The database handles UTF-8 characters cleanly.'); } /** diff --git a/core/modules/system/src/Tests/Database/SchemaTest.php b/core/modules/system/src/Tests/Database/SchemaTest.php index 9a234ed..df1b0b6 100644 --- a/core/modules/system/src/Tests/Database/SchemaTest.php +++ b/core/modules/system/src/Tests/Database/SchemaTest.php @@ -72,7 +72,7 @@ function testSchema() { $columns = db_query('SHOW FULL COLUMNS FROM {test_table}'); foreach ($columns as $column) { if ($column->Field == 'test_field_string') { - $string_check = ($column->Collation == 'utf8_general_ci'); + $string_check = ($column->Collation == 'utf8mb4_general_ci'); } if ($column->Field == 'test_field_string_ascii') { $string_ascii_check = ($column->Collation == 'ascii_general_ci'); diff --git a/core/modules/system/tests/modules/database_test/database_test.install b/core/modules/system/tests/modules/database_test/database_test.install index 7c74c1c..cfd72d5 100644 --- a/core/modules/system/tests/modules/database_test/database_test.install +++ b/core/modules/system/tests/modules/database_test/database_test.install @@ -24,7 +24,7 @@ function database_test_schema() { ), 'name' => array( 'description' => "A person's name", - 'type' => 'varchar', + 'type' => 'varchar_ascii', 'length' => 255, 'not null' => TRUE, 'default' => '', @@ -75,7 +75,7 @@ function database_test_schema() { ), 'job' => array( 'description' => "The person's job", - 'type' => 'varchar', + 'type' => 'varchar_ascii', 'length' => 255, 'not null' => TRUE, 'default' => '', @@ -106,7 +106,7 @@ function database_test_schema() { ), 'job' => array( 'description' => "The person's job", - 'type' => 'varchar', + 'type' => 'varchar_ascii', 'length' => 255, 'not null' => TRUE, 'default' => '', @@ -197,7 +197,7 @@ function database_test_schema() { ), 'name' => array( 'description' => "A person's name.", - 'type' => 'varchar', + 'type' => 'varchar_ascii', 'length' => 255, 'not null' => FALSE, 'default' => '', @@ -228,7 +228,7 @@ function database_test_schema() { ), 'name' => array( 'description' => "A person's name.", - 'type' => 'varchar', + 'type' => 'varchar_ascii', 'length' => 255, 'not null' => FALSE, 'default' => '', diff --git a/core/modules/system/tests/modules/entity_test/src/Entity/EntityTestStringId.php b/core/modules/system/tests/modules/entity_test/src/Entity/EntityTestStringId.php index 7e057d4..d6b6c59 100644 --- a/core/modules/system/tests/modules/entity_test/src/Entity/EntityTestStringId.php +++ b/core/modules/system/tests/modules/entity_test/src/Entity/EntityTestStringId.php @@ -45,7 +45,10 @@ public static function baseFieldDefinitions(EntityTypeInterface $entity_type) { $fields['id'] = BaseFieldDefinition::create('string') ->setLabel(t('ID')) ->setDescription(t('The ID of the test entity.')) - ->setReadOnly(TRUE); + ->setReadOnly(TRUE) + // In order to work around the InnoDB 191 character limit on utf8mb4 + // primary keys, we set the character set for the field to ASCII. + ->setSetting('is_ascii', TRUE); return $fields; } diff --git a/core/modules/user/src/UserStorageSchema.php b/core/modules/user/src/UserStorageSchema.php index 447469d..6247b10 100644 --- a/core/modules/user/src/UserStorageSchema.php +++ b/core/modules/user/src/UserStorageSchema.php @@ -52,6 +52,9 @@ protected function getSharedTableFieldSchema(FieldStorageDefinitionInterface $st // Improves the performance of the user__name index defined // in getEntitySchema(). $schema['fields'][$field_name]['not null'] = TRUE; + // Make sure the field is no longer than 191 characters so we can + // add a unique constraint in MySQL. + $schema['fields'][$field_name]['length'] = USERNAME_MAX_LENGTH; break; case 'mail': diff --git a/core/modules/user/user.module b/core/modules/user/user.module index d175edf..e55eb56 100644 --- a/core/modules/user/user.module +++ b/core/modules/user/user.module @@ -25,6 +25,8 @@ /** * Maximum length of username text field. + * + * Keep this under 191 characters so we can use a unique constraint in MySQL. */ const USERNAME_MAX_LENGTH = 60; diff --git a/core/modules/views/src/Tests/ViewTestData.php b/core/modules/views/src/Tests/ViewTestData.php index 4b5340e..34fcd44 100644 --- a/core/modules/views/src/Tests/ViewTestData.php +++ b/core/modules/views/src/Tests/ViewTestData.php @@ -75,7 +75,7 @@ public static function schemaDefinition() { ), 'name' => array( 'description' => "A person's name", - 'type' => 'varchar', + 'type' => 'varchar_ascii', 'length' => 255, 'not null' => TRUE, 'default' => '', diff --git a/core/scripts/dump-database-d6.sh b/core/scripts/dump-database-d6.sh index ace93d9..d0a89d7 100644 --- a/core/scripts/dump-database-d6.sh +++ b/core/scripts/dump-database-d6.sh @@ -53,6 +53,11 @@ $schema = drupal_get_schema(); ksort($schema); +// Override the field type of the filename primary key to bypass the +// InnoDB 191 character limitation. +if (isset($schema['system']['primary key']) && $schema['system']['primary key'] == 'filename' && isset($schema['system']['fields']['filename']['type']) && $schema['system']['fields']['filename']['type'] == 'varchar') { + $schema['system']['fields']['filename']['type'] = 'varchar_ascii'; +} // Export all the tables in the schema. foreach ($schema as $table => $data) { // Remove descriptions to save time and code. diff --git a/core/tests/Drupal/Tests/Core/Routing/RoutingFixtures.php b/core/tests/Drupal/Tests/Core/Routing/RoutingFixtures.php index ffbfb03..f0aba22 100644 --- a/core/tests/Drupal/Tests/Core/Routing/RoutingFixtures.php +++ b/core/tests/Drupal/Tests/Core/Routing/RoutingFixtures.php @@ -176,7 +176,7 @@ public function routingTableDefinition() { 'fields' => array( 'name' => array( 'description' => 'Primary Key: Machine name of this route', - 'type' => 'varchar', + 'type' => 'varchar_ascii', 'length' => 255, 'not null' => TRUE, 'default' => '', diff --git a/sites/default/default.settings.php b/sites/default/default.settings.php index 3783d0a..f9a5a1b 100644 --- a/sites/default/default.settings.php +++ b/sites/default/default.settings.php @@ -75,7 +75,7 @@ * 'host' => 'localhost', * 'port' => 3306, * 'prefix' => 'myprefix_', - * 'collation' => 'utf8_general_ci', + * 'collation' => 'utf8mb4_general_ci', * ); * @endcode * @@ -127,7 +127,7 @@ * 'password' => 'password', * 'host' => 'localhost', * 'prefix' => 'main_', - * 'collation' => 'utf8_general_ci', + * 'collation' => 'utf8mb4_general_ci', * ); * @endcode *