diff --git a/core/lib/Drupal/Component/Utility/Unicode.php b/core/lib/Drupal/Component/Utility/Unicode.php index 4938b0d..d4dba04 100644 --- a/core/lib/Drupal/Component/Utility/Unicode.php +++ b/core/lib/Drupal/Component/Utility/Unicode.php @@ -697,4 +697,29 @@ public static function validateUtf8($text) { return (preg_match('/^./us', $text) == 1); } + /** + * Finds the position of the first occurrence of a string in another string. + * + * @param string $haystack + * The string to search in. + * @param string $needle + * The string to find in $haystack. + * @param int $offset + * If specified, start the search at this number of characters from the + * beginning (default 0). + * + * @return int|false + * The position where $needle occurs in $haystack, always relative to the + * beginning (independent of $offset), or FALSE if not found. Note that + * a return value of 0 is not the same as FALSE. + */ + public static function strpos($haystack, $needle, $offset = 0) { + if (static::getStatus() == static::STATUS_MULTIBYTE) { + return mb_strpos($haystack, $needle, $offset); + } + else { + return strpos($haystack, $needle, $offset); + } + } + } diff --git a/core/modules/search/search.module b/core/modules/search/search.module index 14ecb23..3fbc1c2 100644 --- a/core/modules/search/search.module +++ b/core/modules/search/search.module @@ -264,7 +264,7 @@ function search_simplify($text, $langcode = NULL) { // Lowercase $text = Unicode::strtolower($text); - // Remove diacitics. + // Remove diacritics. $text = \Drupal::service('transliteration')->removeDiacritics($text); // Call an external processor for word handling. @@ -385,10 +385,22 @@ function search_index_split($text, $langcode = NULL) { * Helper function for array_walk in search_index_split. */ function _search_index_truncate(&$text) { + // Use a static array to avoid re-truncating text we've done before. + // The same words may often be passed in during excerpt generation. + static $truncated = array(); + if (isset($truncated[$text])) { + $text = $truncated[$text]; + return; + } + + // If we didn't find it in the static array, perform the operation. + $original = $text; if (is_numeric($text)) { $text = ltrim($text, '0'); } $text = Unicode::truncate($text, 50); + // Save it for the next time. + $truncated[$original] = $text; } /** @@ -634,7 +646,9 @@ function search_mark_for_reindex($type = NULL, $sid = NULL, $langcode = NULL) { */ function search_excerpt($keys, $text, $langcode = NULL) { // We highlight around non-indexable or CJK characters. - $boundary = '(?:(?<=[' . Unicode::PREG_CLASS_WORD_BOUNDARY . PREG_CLASS_CJK . '])|(?=[' . Unicode::PREG_CLASS_WORD_BOUNDARY . PREG_CLASS_CJK . ']))'; + $boundary_character = '[' . Unicode::PREG_CLASS_WORD_BOUNDARY . PREG_CLASS_CJK . ']'; + $preceded_by_boundary = '(?<=' . $boundary_character . ')'; + $followed_by_boundary = '(?=' . $boundary_character . ')'; // Extract positive keywords and phrases. preg_match_all('/ ("([^"]+)"|(?!OR)([^" ]+))/', ' ' . $keys, $matches); @@ -650,7 +664,7 @@ function search_excerpt($keys, $text, $langcode = NULL) { // search_simplify(). $temp_keys = array(); foreach ($keys as $key) { - $key = _search_find_match_with_simplify($key, $text, $boundary, $langcode); + $key = _search_find_match_with_simplify($key, $text, $boundary_character, $langcode); if (isset($key)) { // Quote slashes so they can be used in regular expressions. $temp_keys[] = preg_quote($key, '/'); @@ -685,7 +699,7 @@ function search_excerpt($keys, $text, $langcode = NULL) { // we are requiring a match on a word boundary, make sure $text starts // and ends with a space. $matches = array(); - if (preg_match('/' . $boundary . $key . $boundary . '/iu', ' ' . $text . ' ', $matches, PREG_OFFSET_CAPTURE, $look_start[$key])) { + if (preg_match('/' . $preceded_by_boundary . $key . $followed_by_boundary . '/iu', ' ' . $text . ' ', $matches, PREG_OFFSET_CAPTURE, $look_start[$key])) { $found_position = $matches[0][1]; $look_start[$key] = $found_position + 1; // Keep track of which keys we found this time, in case we need to @@ -771,7 +785,7 @@ function search_excerpt($keys, $text, $langcode = NULL) { // Highlight keywords. Must be done at once to prevent conflicts ('strong' // and ''). - $text = trim(preg_replace('/' . $boundary . '(?:' . implode('|', $keys) . ')' . $boundary . '/iu', '\0', ' ' . $text . ' ')); + $text = trim(preg_replace('/' . $preceded_by_boundary . '(?:' . implode('|', $keys) . ')' . $followed_by_boundary . '/iu', '\0', ' ' . $text . ' ')); return [ '#markup' => $text, '#allowed_tags' => ['strong'] @@ -790,7 +804,8 @@ function search_excerpt($keys, $text, $langcode = NULL) { * @param string $text * The text to search for the keyword. * @param string $boundary - * Regular expression for boundary characters between words. + * Regular expression for the boundary character class (characters that + * indicate spaces between words). * @param string|null $langcode * Language code for the language of $text, if known. * @@ -801,6 +816,9 @@ function search_excerpt($keys, $text, $langcode = NULL) { * not located, NULL is returned. */ function _search_find_match_with_simplify($key, $text, $boundary, $langcode = NULL) { + $preceded_by_boundary = '(?<=' . $boundary . ')'; + $followed_by_boundary = '(?=' . $boundary . ')'; + // See if $key appears as-is. When testing, make sure $text starts/ends with // a space, because we require $key to be surrounded by word boundary // characters. @@ -808,8 +826,19 @@ function _search_find_match_with_simplify($key, $text, $boundary, $langcode = NU if ($temp == '') { return NULL; } - if (preg_match('/' . $boundary . preg_quote($temp, '/') . $boundary . '/iu', ' ' . $text . ' ')) { - return $key; + if (preg_match('/' . $preceded_by_boundary . preg_quote($temp, '/') . $followed_by_boundary . '/iu', ' ' . $text . ' ')) { + return $temp; + } + + // See if there is a match after lower-casing and removing diacritics in + // both, which should preserve the string length. + $new_text = Unicode::strtolower($text); + $new_text = \Drupal::service('transliteration')->removeDiacritics($new_text); + $new_key = Unicode::strtolower($temp); + $new_key = \Drupal::service('transliteration')->removeDiacritics($new_key); + if (preg_match('/' . $preceded_by_boundary . preg_quote($new_key, '/') . $followed_by_boundary . '/u', ' ' . $new_text . ' ')) { + $position = Unicode::strpos($new_text, $new_key); + return Unicode::substr($text, $position, Unicode::strlen($new_key)); } // Run both text and key through search_simplify. @@ -821,7 +850,7 @@ function _search_find_match_with_simplify($key, $text, $boundary, $langcode = NU } // Split $text into words, keeping track of where the word boundaries are. - $words = preg_split('/' . $boundary . '/iu', $text, NULL, PREG_SPLIT_OFFSET_CAPTURE); + $words = preg_split('/' . $boundary . '+/u', $text, NULL, PREG_SPLIT_OFFSET_CAPTURE); // Add an entry pointing to the end of the string, for the loop below. $words[] = array('', strlen($text)); $num_words = count($words); diff --git a/core/modules/search/src/Tests/SearchExcerptTest.php b/core/modules/search/src/Tests/SearchExcerptTest.php index 9f4a5ad2..53922ab 100644 --- a/core/modules/search/src/Tests/SearchExcerptTest.php +++ b/core/modules/search/src/Tests/SearchExcerptTest.php @@ -35,35 +35,33 @@ class SearchExcerptTest extends WebTestBase { function testSearchExcerpt() { // Make some text with entities and tags. $text = 'The quick brown fox & jumps

over

the lazy dog'; - // Note: The search_excerpt() function adds some extra spaces -- not - // important for HTML formatting. Remove these for comparison. $expected = 'The quick brown fox & jumps over the lazy dog'; - $result = preg_replace('| +|', ' ', $this->doSearchExcerpt('nothing', $text)); + $result = $this->doSearchExcerpt('nothing', $text); $this->assertEqual(preg_replace('| +|', ' ', $result), $expected, 'Entire string, stripped of HTML tags, is returned when keyword is not found in short string'); - $result = preg_replace('| +|', ' ', $this->doSearchExcerpt('fox', $text)); + $result = $this->doSearchExcerpt('fox', $text); $this->assertEqual($result, 'The quick brown fox & jumps over the lazy dog', 'Found keyword is highlighted'); $expected = 'The quick brown fox & jumps over the lazy dog'; - $result = preg_replace('| +|', ' ', $this->doSearchExcerpt('The', $text)); + $result = $this->doSearchExcerpt('The', $text); $this->assertEqual(preg_replace('| +|', ' ', $result), $expected, 'Keyword is highlighted at beginning of short string'); $expected = 'The quick brown fox & jumps over the lazy dog'; - $result = preg_replace('| +|', ' ', $this->doSearchExcerpt('dog', $text)); + $result = $this->doSearchExcerpt('dog', $text); $this->assertEqual(preg_replace('| +|', ' ', $result), $expected, 'Keyword is highlighted at end of short string'); $longtext = str_repeat(str_replace('brown', 'silver', $text) . ' ', 10) . $text . str_repeat(' ' . str_replace('brown', 'pink', $text), 10); - $result = preg_replace('| +|', ' ', $this->doSearchExcerpt('brown', $longtext)); + $result = $this->doSearchExcerpt('brown', $longtext); $expected = '… silver fox & jumps over the lazy dog The quick brown fox & jumps over the lazy dog The quick …'; $this->assertEqual($result, $expected, 'Snippet around keyword in long text is correctly capped'); $longtext = str_repeat($text . ' ', 10); - $result = preg_replace('| +|', ' ', $this->doSearchExcerpt('nothing', $longtext)); + $result = $this->doSearchExcerpt('nothing', $longtext); $expected = 'The quick brown fox & jumps over the lazy dog'; $this->assertTrue(strpos($result, $expected) === 0, 'When keyword is not found in long string, return value starts as expected'); $entities = str_repeat('készítése ', 20); - $result = preg_replace('| +|', ' ', $this->doSearchExcerpt('nothing', $entities)); + $result = $this->doSearchExcerpt('nothing', $entities); $this->assertFalse(strpos($result, '&'), 'Entities are not present in excerpt'); $this->assertTrue(strpos($result, 'í') > 0, 'Entities are converted in excerpt'); @@ -82,6 +80,8 @@ function testSearchExcerpt() { * and compares them with strings that contain the original unsimplified word. */ function testSearchExcerptSimplified() { + $start_time = microtime(TRUE); + $lorem1 = 'Lorem ipsum dolor sit amet, consectetur adipiscing elit. Etiam vitae arcu at leo cursus laoreet. Curabitur dui tortor, adipiscing malesuada tempor in, bibendum ac diam. Cras non tellus a libero pellentesque condimentum. What is a Drupalism? Suspendisse ac lacus libero. Ut non est vel nisl faucibus interdum nec sed leo. Pellentesque sem risus, vulputate eu semper eget, auctor in libero.'; $lorem2 = 'Ut fermentum est vitae metus convallis scelerisque. Phasellus pellentesque rhoncus tellus, eu dignissim purus posuere id. Quisque eu fringilla ligula. Morbi ullamcorper, lorem et mattis egestas, tortor neque pretium velit, eget eleifend odio turpis eu purus. Donec vitae metus quis leo pretium tincidunt a pulvinar sem. Morbi adipiscing laoreet mauris vel placerat. Nullam elementum, nisl sit amet scelerisque malesuada, dolor nunc hendrerit quam, eu ultrices erat est in orci.'; @@ -89,37 +89,37 @@ function testSearchExcerptSimplified() { $text = $lorem1 . ' Number: 123456.7890 Hyphenated: one-two abc,def ' . $lorem2; // Note: The search_excerpt() function adds some extra spaces -- not // important for HTML formatting. Remove these for comparison. - $result = preg_replace('| +|', ' ', $this->doSearchExcerpt('123456.7890', $text)); + $result = $this->doSearchExcerpt('123456.7890', $text); $this->assertTrue(strpos($result, 'Number: 123456.7890') !== FALSE, 'Numeric keyword is highlighted with exact match'); - $result = preg_replace('| +|', ' ', $this->doSearchExcerpt('1234567890', $text)); + $result = $this->doSearchExcerpt('1234567890', $text); $this->assertTrue(strpos($result, 'Number: 123456.7890') !== FALSE, 'Numeric keyword is highlighted with simplified match'); - $result = preg_replace('| +|', ' ', $this->doSearchExcerpt('Number 1234567890', $text)); + $result = $this->doSearchExcerpt('Number 1234567890', $text); $this->assertTrue(strpos($result, 'Number: 123456.7890') !== FALSE, 'Punctuated and numeric keyword is highlighted with simplified match'); - $result = preg_replace('| +|', ' ', $this->doSearchExcerpt('"Number 1234567890"', $text)); + $result = $this->doSearchExcerpt('"Number 1234567890"', $text); $this->assertTrue(strpos($result, 'Number: 123456.7890') !== FALSE, 'Phrase with punctuated and numeric keyword is highlighted with simplified match'); - $result = preg_replace('| +|', ' ', $this->doSearchExcerpt('"Hyphenated onetwo"', $text)); + $result = $this->doSearchExcerpt('"Hyphenated onetwo"', $text); $this->assertTrue(strpos($result, 'Hyphenated: one-two') !== FALSE, 'Phrase with punctuated and hyphenated keyword is highlighted with simplified match'); - $result = preg_replace('| +|', ' ', $this->doSearchExcerpt('"abc def"', $text)); + $result = $this->doSearchExcerpt('"abc def"', $text); $this->assertTrue(strpos($result, 'abc,def') !== FALSE, 'Phrase with keyword simplified into two separate words is highlighted with simplified match'); // Test phrases with characters which are being truncated. - $result = preg_replace('| +|', ' ', $this->doSearchExcerpt('"ipsum _"', $text)); + $result = $this->doSearchExcerpt('"ipsum _"', $text); $this->assertTrue(strpos($result, 'ipsum') !== FALSE, 'Only valid part of the phrase is highlighted and invalid part containing "_" is ignored.'); - $result = preg_replace('| +|', ' ', $this->doSearchExcerpt('"ipsum 0000"', $text)); + $result = $this->doSearchExcerpt('"ipsum 0000"', $text); $this->assertTrue(strpos($result, 'ipsum') !== FALSE, 'Only valid part of the phrase is highlighted and invalid part "0000" is ignored.'); // Test combination of the valid keyword and keyword containing only // characters which are being truncated during simplification. - $result = preg_replace('| +|', ' ', $this->doSearchExcerpt('ipsum _', $text)); + $result = $this->doSearchExcerpt('ipsum _', $text); $this->assertTrue(strpos($result, 'ipsum') !== FALSE, 'Only valid keyword is highlighted and invalid keyword "_" is ignored.'); - $result = preg_replace('| +|', ' ', $this->doSearchExcerpt('ipsum 0000', $text)); + $result = $this->doSearchExcerpt('ipsum 0000', $text); $this->assertTrue(strpos($result, 'ipsum') !== FALSE, 'Only valid keyword is highlighted and invalid keyword "0000" is ignored.'); // Test using the hook_search_preprocess() from the test module. @@ -127,37 +127,51 @@ function testSearchExcerptSimplified() { // So, if we search for "find" or "finds" or "finding", we should // highlight "finding". $text = "this tests finding a string"; - $result = preg_replace('| +|', ' ', $this->doSearchExcerpt('finds', $text, 'ex')); + $result = $this->doSearchExcerpt('finds', $text, 'ex'); $this->assertTrue(strpos($result, 'finding') !== FALSE, 'Search excerpt works with preprocess hook, search for finds'); - $result = preg_replace('| +|', ' ', $this->doSearchExcerpt('find', $text, 'ex')); + $result = $this->doSearchExcerpt('find', $text, 'ex'); $this->assertTrue(strpos($result, 'finding') !== FALSE, 'Search excerpt works with preprocess hook, search for find'); // Just to be sure, test with the replacement at the beginning and end. $text = "finding at the beginning"; - $result = preg_replace('| +|', ' ', $this->doSearchExcerpt('finds', $text, 'ex')); + $result = $this->doSearchExcerpt('finds', $text, 'ex'); $this->assertTrue(strpos($result, 'finding') !== FALSE, 'Search excerpt works with preprocess hook, text at start'); $text = "at the end finding"; - $result = preg_replace('| +|', ' ', $this->doSearchExcerpt('finds', $text, 'ex')); + $result = $this->doSearchExcerpt('finds', $text, 'ex'); $this->assertTrue(strpos($result, 'finding') !== FALSE, 'Search excerpt works with preprocess hook, text at end'); // Testing with a one-to-many replacement: the test module replaces DIC // with Dependency Injection Container. $text = "something about the DIC is happening"; - $result = preg_replace('| +|', ' ', $this->doSearchExcerpt('Dependency', $text, 'ex')); + $result = $this->doSearchExcerpt('Dependency', $text, 'ex'); $this->assertTrue(strpos($result, 'DIC') !== FALSE, 'Search excerpt works with preprocess hook, acronym first word'); - $result = preg_replace('| +|', ' ', $this->doSearchExcerpt('Injection', $text, 'ex')); + $result = $this->doSearchExcerpt('Injection', $text, 'ex'); $this->assertTrue(strpos($result, 'DIC') !== FALSE, 'Search excerpt works with preprocess hook, acronym second word'); - $result = preg_replace('| +|', ' ', $this->doSearchExcerpt('Container', $text, 'ex')); + $result = $this->doSearchExcerpt('Container', $text, 'ex'); $this->assertTrue(strpos($result, 'DIC') !== FALSE, 'Search excerpt works with preprocess hook, acronym third word'); // Testing with a many-to-one replacement: the test module replaces // hypertext markup language with HTML. $text = "we always use hypertext markup language to describe things"; - $result = preg_replace('| +|', ' ', $this->doSearchExcerpt('html', $text, 'ex')); + $result = $this->doSearchExcerpt('html', $text, 'ex'); $this->assertTrue(strpos($result, 'hypertext markup language') !== FALSE, 'Search excerpt works with preprocess hook, acronym many to one'); + + // Test with accents and caps in a longer piece of text with the target + // near the end. + $text = str_repeat($lorem2, 20) . ' ' . $lorem1; + $result = $this->doSearchExcerpt('Lìbêró', $text); + $this->assertTrue(strpos($result, 'libero') !== FALSE, 'Search excerpt works with caps and accents in longer text'); + + // Test with an acronym provided by the hook, with the target text in the + // middle of a long string. + $text = str_repeat($lorem2, 10) . ' DIC ' . str_repeat($lorem2, 10); + $result = $this->doSearchExcerpt('Dependency', $text, 'ex'); + $this->assertTrue(strpos($result, 'DIC') !== FALSE, 'Search excerpt works with acronym in longer text'); + + $this->verbose('Elapsed time: ' . (microtime(TRUE) - $start_time)); } /** @@ -175,7 +189,10 @@ function testSearchExcerptSimplified() { */ protected function doSearchExcerpt($keys, $render_array, $langcode = NULL) { $render_array = search_excerpt($keys, $render_array, $langcode); - return \Drupal::service('renderer')->renderPlain($render_array); + $text = \Drupal::service('renderer')->renderPlain($render_array); + // The search_excerpt() function adds some extra spaces -- not + // important for HTML formatting or this test. Remove these for comparison. + return preg_replace('| +|', ' ', $text); } }