diff --git a/includes/database/database.inc b/includes/database/database.inc index 90a3f74..d478f71 100644 --- a/includes/database/database.inc +++ b/includes/database/database.inc @@ -1313,6 +1313,28 @@ abstract class DatabaseConnection extends PDO { * also larger than the $existing_id if one was passed in. */ abstract public function nextId($existing_id = 0); + + /** + * Checks whether utf8mb4 support is available on the current database system. + * + * @return bool + */ + public function utf8mb4IsSupported() { + // By default we assume that the database backend may not support 4 byte + // UTF-8. + return FALSE; + } + + /** + * Checks whether utf8mb4 support is enabled in settings.php. + * + * @return bool + */ + public function utf8mb4IsEnabled() { + // Since 4 byte UTF-8 is not supported by default, there is nothing to + // enable. + return FALSE; + } } /** diff --git a/includes/database/mysql/database.inc b/includes/database/mysql/database.inc index a96b053..036e301 100644 --- a/includes/database/mysql/database.inc +++ b/includes/database/mysql/database.inc @@ -28,6 +28,18 @@ class DatabaseConnection_mysql extends DatabaseConnection { $this->connectionOptions = $connection_options; + $charset = 'utf8'; + // Check if the charset is overridden to utf8mb4 in settings.php. + if ($this->utf8mb4IsEnabled()) { + $charset = 'utf8mb4'; + } + // Fall back to utf8 if the internal '_dsn_utf8_fallback' flag is set while + // running the database tasks in the installer. + if (isset($this->connectionOptions['_dsn_utf8_fallback']) && $this->connectionOptions['_dsn_utf8_fallback'] === TRUE) { + $charset = 'utf8'; + $connection_options['collation'] = 'utf8_general_ci'; + } + // The DSN should use either a socket or a host/port. if (isset($connection_options['unix_socket'])) { $dsn = 'mysql:unix_socket=' . $connection_options['unix_socket']; @@ -39,7 +51,7 @@ class DatabaseConnection_mysql extends DatabaseConnection { // Character set is added to dsn to ensure PDO uses the proper character // set when escaping. This has security implications. See // https://www.drupal.org/node/1201452 for further discussion. - $dsn .= ';charset=utf8'; + $dsn .= ';charset=' . $charset; $dsn .= ';dbname=' . $connection_options['database']; // Allow PDO options to be overridden. $connection_options += array( @@ -63,10 +75,10 @@ class DatabaseConnection_mysql extends DatabaseConnection { // certain one has been set; otherwise, MySQL defaults to 'utf8_general_ci' // for UTF-8. if (!empty($connection_options['collation'])) { - $this->exec('SET NAMES utf8 COLLATE ' . $connection_options['collation']); + $this->exec('SET NAMES ' . $charset . ' COLLATE ' . $connection_options['collation']); } else { - $this->exec('SET NAMES utf8'); + $this->exec('SET NAMES ' . $charset); } // Set MySQL init_commands if not already defined. Default Drupal's MySQL @@ -206,6 +218,44 @@ class DatabaseConnection_mysql extends DatabaseConnection { } } } + + /** + * {@inheritdoc} + */ + public function utf8mb4IsEnabled() { + return isset($this->connectionOptions['charset']) && $this->connectionOptions['charset'] === 'utf8mb4'; + } + + /** + * {@inheritdoc} + */ + public function utf8mb4IsSupported() { + // Ensure that the MySQL driver supports utf8mb4 encoding. + $version = $this->getAttribute(\PDO::ATTR_CLIENT_VERSION); + if (FALSE !== strpos($version, 'mysqlnd')) { + // The mysqlnd driver supports utf8mb4 starting at version 5.0.9. + $version = preg_replace('/^\D+([\d.]+).*/', '$1', $version); + if (version_compare($version, '5.0.9', '<')) { + return FALSE; + } + } + else { + // The libmysqlclient driver supports utf8mb4 starting at version 5.5.3. + if (version_compare($version, '5.5.3', '<')) { + return FALSE; + } + } + + // Ensure that the MySQL server supports large prefixes and utf8mb4. + try { + $this->query("CREATE TABLE {drupal_utf8mb4_test} (id VARCHAR(255), PRIMARY KEY(id(255))) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci ROW_FORMAT=DYNAMIC"); + } + catch (\Exception $e) { + return FALSE; + } + $this->query("DROP TABLE {drupal_utf8mb4_test}"); + return TRUE; + } } diff --git a/includes/database/mysql/install.inc b/includes/database/mysql/install.inc index 75f2ae3..2056d4f 100644 --- a/includes/database/mysql/install.inc +++ b/includes/database/mysql/install.inc @@ -17,6 +17,11 @@ class DatabaseTasks_mysql extends DatabaseTasks { protected $pdoDriver = 'mysql'; /** + * Error code for "Can't initialize character set" error. + */ + const UNSUPPORTED_CHARSET = 2019; + + /** * Returns a human-readable name string for MySQL and equivalent databases. */ public function name() { @@ -29,5 +34,51 @@ class DatabaseTasks_mysql extends DatabaseTasks { public function minimumVersion() { return '5.0.15'; } + + /** + * Check if we can connect to the database. + */ + protected function connect() { + try { + // This doesn't actually test the connection. + db_set_active(); + // Now actually do a check. + try { + Database::getConnection(); + } + catch (\Exception $e) { + // Detect utf8mb4 incompatibility. + // Error code for "Can't initialize character set" error. + if ($e->getCode() == self::UNSUPPORTED_CHARSET) { + $this->fail(t('Your MySQL server and PHP MySQL driver must support utf8mb4 character encoding. Make sure to use a database system that supports this (such as MySQL/MariaDB/Percona 5.5.3 and up), and that the utf8mb4 character set is compiled in. See the MySQL documentation for more information.', array('@documentation' => 'https://dev.mysql.com/doc/refman/5.0/en/cannot-initialize-character-set.html'))); + $info = Database::getConnectionInfo(); + $info_copy = $info; + // Set a flag to fall back to utf8. Note: this flag should only be + // used here and is for internal use only. + $info_copy['default']['_dsn_utf8_fallback'] = TRUE; + // In order to change the Database::$databaseInfo array, we need to + // remove the active connection, then re-add it with the new info. + Database::removeConnection('default'); + Database::addConnectionInfo('default', 'default', $info_copy['default']); + // Connect with the new database info, using the utf8 character set so + // that we can run the checkEngineVersion test. + Database::getConnection(); + // Revert to the old settings. + Database::removeConnection('default'); + Database::addConnectionInfo('default', 'default', $info['default']); + } + else { + // Rethrow the exception. + throw $e; + } + } + $this->pass('Drupal can CONNECT to the database ok.'); + } + catch (Exception $e) { + $this->fail(st('Failed to connect to your database server. The server reports the following message: %error.', array('%error' => $e->getMessage()))); + return FALSE; + } + return TRUE; + } } diff --git a/includes/database/mysql/schema.inc b/includes/database/mysql/schema.inc index 2a2722e..f8fea1a 100644 --- a/includes/database/mysql/schema.inc +++ b/includes/database/mysql/schema.inc @@ -81,7 +81,8 @@ class DatabaseSchema_mysql extends DatabaseSchema { // Provide defaults if needed. $table += array( 'mysql_engine' => 'InnoDB', - 'mysql_character_set' => 'utf8', + // Allow the default charset to be overridden in settings.php. + 'mysql_character_set' => $this->connection->utf8mb4IsEnabled() ? 'utf8mb4' : 'utf8', ); $sql = "CREATE TABLE {" . $name . "} (\n"; @@ -109,6 +110,13 @@ class DatabaseSchema_mysql extends DatabaseSchema { $sql .= ' COLLATE ' . $info['collation']; } + // The row format needs to be either DYNAMIC or COMPRESSED in order to allow + // for the innodb_large_prefix setting to take effect, see + // https://dev.mysql.com/doc/refman/5.6/en/create-table.html + if ($this->connection->utf8mb4IsEnabled()) { + $sql .= ' ROW_FORMAT=DYNAMIC'; + } + // Add table comment. if (!empty($table['description'])) { $sql .= ' COMMENT ' . $this->prepareComment($table['description'], self::COMMENT_MAX_TABLE); diff --git a/includes/database/pgsql/database.inc b/includes/database/pgsql/database.inc index 67b49fe..24af658 100644 --- a/includes/database/pgsql/database.inc +++ b/includes/database/pgsql/database.inc @@ -216,6 +216,20 @@ class DatabaseConnection_pgsql extends DatabaseConnection { return $id; } + + /** + * {@inheritdoc} + */ + public function utf8mb4IsEnabled() { + return TRUE; + } + + /** + * {@inheritdoc} + */ + public function utf8mb4IsSupported() { + return TRUE; + } } /** diff --git a/includes/database/sqlite/database.inc b/includes/database/sqlite/database.inc index 8a5ba8c..bdb2a5e 100644 --- a/includes/database/sqlite/database.inc +++ b/includes/database/sqlite/database.inc @@ -512,6 +512,20 @@ class DatabaseStatement_sqlite extends DatabaseStatementPrefetch implements Iter return $return; } + + /** + * {@inheritdoc} + */ + public function utf8mb4IsEnabled() { + return TRUE; + } + + /** + * {@inheritdoc} + */ + public function utf8mb4IsSupported() { + return TRUE; + } } /** diff --git a/includes/install.core.inc b/includes/install.core.inc index ad43b42..0364cf8 100644 --- a/includes/install.core.inc +++ b/includes/install.core.inc @@ -851,6 +851,12 @@ function install_verify_settings() { if (empty($errors)) { return TRUE; } + else { + foreach ($errors as $name => $message) { + include('includes/form.inc'); + form_set_error($name, $message); + } + } } return FALSE; } diff --git a/modules/node/node.test b/modules/node/node.test index c7c4711..a751e69 100644 --- a/modules/node/node.test +++ b/modules/node/node.test @@ -2986,3 +2986,39 @@ class NodePageCacheTest extends NodeWebTestCase { $this->assertResponse(404); } } + +/** + * Tests that we store and retrieve multi-byte UTF-8 characters correctly. + */ +class NodeMultiByteUtf8Test extends NodeWebTestCase { + + /** + * {@inheritdoc} + */ + public static function getInfo() { + return array( + 'name' => 'Multi-byte UTF-8', + 'description' => 'Test that we store and retrieve multi-byte UTF-8 characters correctly.', + 'group' => 'Node', + ); + } + + /** + * Tests that we store and retrieve multi-byte UTF-8 characters correctly. + */ + public function testMultiByteUtf8() { + $connection = Database::getConnection(); + // On MySQL, this test will only run if 'charset' is set to 'utf8mb4' in + // settings.php. + if ($connection->driver() == 'mysql' && !($connection->utf8mb4IsSupported() && $connection->utf8mb4IsEnabled())) { + return; + } + $title = '🐙'; + $this->assertTrue(mb_strlen($title, 'utf-8') < strlen($title), 'Title has multi-byte characters.'); + $node = $this->drupalCreateNode(array('title' => $title)); + $this->drupalGet('node/' . $node->nid); + $result = $this->xpath('//h1[@id="page-title"]'); + $this->assertEqual(trim((string) $result[0]), $title, 'The passed title was returned.'); + } + +} diff --git a/modules/system/system.install b/modules/system/system.install index 323b7b3..cd3365e 100644 --- a/modules/system/system.install +++ b/modules/system/system.install @@ -196,6 +196,12 @@ function system_requirements($phase) { ); } + // Test database-specific multi-byte UTF-8 related requirements. + $charset_requirements = _system_check_db_utf8mb4_requirements(); + if (!empty($charset_requirements)) { + $requirements['database_charset'] = $charset_requirements; + } + // Test PHP memory_limit $memory_limit = ini_get('memory_limit'); $requirements['php_memory_limit'] = array( @@ -518,6 +524,54 @@ function system_requirements($phase) { } /** + * Checks whether the requirements for multi-byte UTF-8 support are met. + * + * @return array + * A requirements array with the result of the charset check. + */ +function _system_check_db_utf8mb4_requirements() { + global $install_state; + // Skip the requirements check if we're in the initial step of the installer + // where the settings have not yet been verified. + if ($install_state['settings_verified'] === FALSE) { + return array(); + } + $connection = Database::getConnection(); + $t = get_t(); + $requirements['title'] = $t('Database 4 byte UTF-8 support'); + + $utf8mb4_enabled = $connection->utf8mb4IsEnabled(); + $utf8mb4_supported = $connection->utf8mb4IsSupported(); + $driver = $connection->driver(); + + if ($utf8mb4_enabled) { + if ($utf8mb4_supported) { + $requirements['value'] = $t('Enabled'); + $requirements['description'] = $t('4 byte UTF-8 for @driver is enabled.', array('@driver' => $driver)); + $requirements['severity'] = REQUIREMENT_OK; + } + else { + $requirements['value'] = $t('Not supported'); + $requirements['description'] = $t('4 byte UTF-8 for @driver is activated in settings.php, but not supported on your system. Please turn this off in settings.php, or ensure that all requirements are met. See https://www.drupal.org/node/2754539 for more information.', array('@driver' => $driver)); + $requirements['severity'] = REQUIREMENT_ERROR; + } + } + else { + if ($utf8mb4_supported) { + $requirements['value'] = $t('Not enabled'); + $requirements['description'] = $t('4 byte UTF-8 for @driver is not activated in settings.php, but it is supported on your system. It is highly recommended you enable this. See https://www.drupal.org/node/2754539 for more information.', array('@driver' => $driver)); + $requirements['severity'] = REQUIREMENT_WARNING; + } + else { + $requirements['value'] = $t('Disabled'); + $requirements['description'] = $t('4 byte UTF-8 for @driver is disabled. See https://www.drupal.org/node/2754539 for more information.', array('@driver' => $driver)); + $requirements['severity'] = REQUIREMENT_INFO; + } + } + return $requirements; +} + +/** * Implements hook_install(). */ function system_install() { diff --git a/sites/default/default.settings.php b/sites/default/default.settings.php index 7e36a4a..2121f2e 100644 --- a/sites/default/default.settings.php +++ b/sites/default/default.settings.php @@ -126,6 +126,37 @@ * ); * @endcode * + * For handling full UTF-8 in MySQL, including multi-byte characters such as + * emojis, asian symbols, and mathematical symbols, you may set the collation + * and charset to "utf8mb4" prior to running install.php: + * @code + * $databases['default']['default'] = array( + * 'driver' => 'mysql', + * 'database' => 'databasename', + * 'username' => 'username', + * 'password' => 'password', + * 'host' => 'localhost', + * 'charset' => 'utf8mb4', + * 'collation' => 'utf8mb4_general_ci', + * ); + * @endcode + * When using this setting on an existing installation, ensure that + * all existing tables have been converted to the utf8mb4 charset, + * for example by using the utf8mb4_convert contrib project, so as to prevent + * mixing data with different charsets. + * Note this should only be used when all of the following conditions are met: + * - In order to allow for large indexes, MySQL must be set up with the + * following my.cnf settings: + * [mysqld] + * innodb_large_prefix=true + * innodb_file_format=barracuda + * innodb_file_per_table=true + * These settings are available as of MySQL 5.5.14, and are defaults in + * MySQL 5.7.7 and up. + * - The PHP MySQL driver must support the utf8mb4 charset (libmysqlclient + 5.5.3 and up, as well as mysqlnd 5.0.9 and up). + * - The MySQL server must support the utf8mb4 charset (5.5.3 and up). + * * You can optionally set prefixes for some or all database table names * by using the 'prefix' setting. If a prefix is specified, the table * name will be prepended with its value. Be sure to use valid database