diff --git a/core/lib/Drupal/Component/Transliteration/PhpTransliteration.php b/core/lib/Drupal/Component/Transliteration/PhpTransliteration.php index 7911f64..1ff9d36 100644 --- a/core/lib/Drupal/Component/Transliteration/PhpTransliteration.php +++ b/core/lib/Drupal/Component/Transliteration/PhpTransliteration.php @@ -82,26 +82,24 @@ public function removeDiacritics($string) { $result = ''; foreach (preg_split('//u', $string, 0, PREG_SPLIT_NO_EMPTY) as $character) { - $to_add = $character; $code = self::ordUTF8($character); - // unicode range 0x00bf to 0x017f + // These two Unicode ranges include the accented US-ASCII letters, with a few characters + // that aren't accented letters mixed in. So define the ranges and the excluded characters. $range1 = $code > 0x00bf && $code < 0x017f; - $exclusions_r1_x00 = array(0x00c6, 0x00d0, 0x00d7, 0x00de, 0x00df, 0x00e6, 0x00f0, 0x00f7, 0x00fe); - $exclusions_r1_x01 = array(0x0132, 0x0133, 0x0138, 0x0149, 0x014a, 0x014b, 0x0152, 0x0153); - $exclusions_range1 = array_merge($exclusions_r1_x00, $exclusions_r1_x01); - - // unicode range 01CD to 024F + $exclusions_range1 = array(0x00d0, 0x00d7, 0x00f0, 0x00f7, 0x0138, 0x014a, 0x014b); $range2 = $code > 0x01cc && $code < 0x0250; - $exclusions_r2_x01 = array(0x01DD, 0x01e2, 0x01e3, 0x01ee, 0x01ef, 0x01f1, 0x01f2, 0x01f3, 0x01f6, 0x01f7, 0x01fc, 0x01fd, 0x021c, 0x021d); - $exclusions_r2_x02 = array(0x0220, 0x0221, 0x0222, 0x0223, 0x0238, 0x0239, 0x0241, 0x0242, 0x0245); - $exclusions_range2 = array_merge($exclusions_r2_x01, $exclusions_r2_x02); + $exclusions_range2 = array(0x01DD, 0x01f7, 0x021c, 0x021d, 0x0220, 0x0221, 0x0241, 0x0242, 0x0245); + $replacement = $character; if (($range1 && !in_array($code, $exclusions_range1)) || ($range2 && !in_array($code, $exclusions_range2))) { - $to_add = $this->lookupReplacement($code); + $to_add = $this->lookupReplacement($code, 'xyz'); + if(strlen($to_add) === 1) { + $replacement = $to_add; + } } - $result .= $to_add; + $result .= $replacement; } return $result; @@ -184,7 +182,8 @@ protected static function ordUTF8($character) { * * @return string * US-ASCII replacement character. If it has a mapping, it is returned; - * otherwise, $unknown_character is returned. + * otherwise, $unknown_character is returned. The replacement can contain + * multiple characters. */ protected function replace($code, $langcode, $unknown_character) { if ($code < 0x80) { @@ -204,17 +203,18 @@ protected function replace($code, $langcode, $unknown_character) { } /** - * Lookup the replacement for a certain UTF-8 character code. + * Look up the generic replacement for a UTF-8 character code. * * @param $code - * The UTF-8 character code. + * The UTF-8 character code. * @param string $unknown_character - * (optional) The character to substitute for characters without transliterated - * equivalents. + * (optional) The character to substitute for characters without entries in + * the replacement tables * * @return string - * US-ASCII replacement character. If it has a mapping, it is returned; - * otherwise, $unknown_character is returned. + * US-ASCII replacement characters. If it has a mapping, it is returned; + * otherwise, $unknown_character is returned. The replacement can contain + * multiple characters. */ protected function lookupReplacement($code, $unknown_character = '?') { // See if there is a generic mapping for this character. diff --git a/core/lib/Drupal/Component/Transliteration/TransliterationInterface.php b/core/lib/Drupal/Component/Transliteration/TransliterationInterface.php index 5107bf3..4d088ef 100644 --- a/core/lib/Drupal/Component/Transliteration/TransliterationInterface.php +++ b/core/lib/Drupal/Component/Transliteration/TransliterationInterface.php @@ -19,9 +19,8 @@ * * This only applies to certain letters: Accented Latin characters like * a-with-acute-accent, in the UTF-8 character range of 0xE0 to 0xE6 and - * 01CD to 024F. Replacements that would result in zero or two characters are - * excluded as well as characters that would need afull transliteration - * (e.g. Eth, Multiply, Divide etc.). + * 01CD to 024F. Replacements that would result in the string changing length + * are excluded, as well as characters that are not accented US-ASCII letters. * * @param string $string * The string holding diacritics. @@ -52,5 +51,4 @@ public function removeDiacritics($string); * characters, and unknown characters replaced with $unknown_character. */ public function transliterate($string, $langcode = 'en', $unknown_character = '?', $max_length = NULL); - } diff --git a/core/modules/search/search.module b/core/modules/search/search.module index ac27c25..e5a5199 100644 --- a/core/modules/search/search.module +++ b/core/modules/search/search.module @@ -241,6 +241,9 @@ function search_simplify($text, $langcode = NULL) { // Lowercase $text = Unicode::strtolower($text); + // Remove diacitics. + $text = \Drupal::service('transliteration')->removeDiacritics($text); + // Call an external processor for word handling. search_invoke_preprocess($text, $langcode); @@ -392,9 +395,6 @@ function search_index($type, $sid, $langcode, $text) { $text = str_replace(array('<', '>'), array(' <', '> '), $text); $text = strip_tags($text, '<' . implode('><', array_keys($tags)) . '>'); - // Remove diacitics. - $text = \Drupal::service('transliteration')->removeDiacritics($text); - // Split HTML tags from plain text. $split = preg_split('/\s*<([^>]+?)>\s*/', $text, -1, PREG_SPLIT_DELIM_CAPTURE); // Note: PHP ensures the array consists of alternating delimiters and literals diff --git a/core/tests/Drupal/Tests/Component/Transliteration/PhpTransliterationTest.php b/core/tests/Drupal/Tests/Component/Transliteration/PhpTransliterationTest.php index 4da3bd4..f01ee05 100644 --- a/core/tests/Drupal/Tests/Component/Transliteration/PhpTransliterationTest.php +++ b/core/tests/Drupal/Tests/Component/Transliteration/PhpTransliterationTest.php @@ -26,14 +26,13 @@ class PhpTransliterationTest extends UnitTestCase { * @param string $original * The language code to test. * @param string $expected - * The expected return from PhpTransliteration::removeDiacritics(). + * The expected return from PhpTransliteration::removeDiacritics(). * * @dataProvider providerTestPhpTransliterationRemoveDiacritics */ public function testRemoveDiacritics($original, $expected) { $transliterator_class = new PhpTransliteration(); $result = $transliterator_class->removeDiacritics($original); -// throw new \Exception($result); $this->assertEquals($expected, $result); } @@ -46,7 +45,7 @@ public function testRemoveDiacritics($original, $expected) { */ public function providerTestPhpTransliterationRemoveDiacritics() { return array( - // unicode range 0x00bf to 0x017f + // Test all characters in the Unicode range 0x00bf to 0x017f. array('ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏ', 'AAAAAAÆCEEEEIIII'), array('ÐÑÒÓÔÕÖרÙÚÛÜÝÞß', 'ÐNOOOOO×OUUUUYÞß'), array('àáâãäåæçèéêëìíîï', 'aaaaaaæceeeeiiii'), @@ -60,7 +59,7 @@ public function providerTestPhpTransliterationRemoveDiacritics() { array('ŠšŢţŤťŦŧŨũŪūŬŭŮů', 'SsTtTtTtUuUuUuUu'), array('ŰűŲųŴŵŶŷŸŹźŻżŽž', 'UuUuWwYyYZzZzZz'), - // unicode range 01CD to 024F + // Test all characters in the Unicode range 0x01CD to 0x024F. array('ǍǎǏ', 'AaI'), array('ǐǑǒǓǔǕǖǗǘǙǚǛǜǝǞǟ', 'iOoUuUuUuUuUuǝAa'), array('ǠǡǢǣǤǥǦǧǨǩǪǫǬǭǮǯ', 'AaǢǣGgGgKkOoOoǮǯ'),