Changing the setting below will cause the site index to be rebuilt. The search index is not cleared but systematically updated to reflect the new settings. Searching will continue to work but new content won\'t be indexed until all existing content has been re-indexed.
The default settings should be appropriate for the majority of sites.
') .''; - $group .= form_textfield(t('Minimum word length to index'), 'minimum_word_size', variable_get('minimum_word_size', 3), 5, 3, t('The number of characters a word has to be to be indexed. Words shorter than this will not be searchable.')); - $group .= form_textfield(t('Minimum word length to search for'), 'remove_short', variable_get('remove_short', 3), 5, 3, t('The number of characters a word has to be to be searched for, including wildcard characters.')); + $group .= form_textfield(t('Minimum word length to index'), 'minimum_word_size', variable_get('minimum_word_size', 5), 5, 3, t('The number of characters a word has to be to be indexed. Words shorter than this will not be searchable.')); + $group .= form_textfield(t('Minimum word/phrase length to search for'), 'remove_short', variable_get('remove_short', 3), 5, 3, t('The number of characters a word or phrase has to be to be searched for.')); $output .= form_group(t('Indexing settings'), $group); return system_settings_form($output); @@ -179,6 +186,7 @@ module_invoke_all('search', 'reset'); } else { + db_query("DELETE FROM {search_dataset} WHERE sid = %d AND type = '%s'", $sid, $type); db_query("DELETE FROM {search_index} WHERE sid = %d AND type = '%s'", $sid, $type); db_query("DELETE FROM {search_index} WHERE fromsid = %d AND fromtype = '%s'", $sid, $type); } @@ -212,7 +220,11 @@ } // Update word counts for new/changed words foreach (search_dirty() as $word => $dummy) { + // Get total count $total = db_result(db_query("SELECT SUM(score) FROM {search_index} WHERE word = '%s'", $word)); + // Apply Zipf's law + $total = log10(1 + 1/(max(1, $total))); + // Store IDF db_query("UPDATE {search_total} SET count = %d WHERE word = '%s'", $total, $word); if (!db_affected_rows()) { db_query("INSERT INTO {search_total} (word, count) VALUES ('%s', %d)", $word, $total); @@ -228,22 +240,18 @@ } /** - * Splits a string into component words according to indexing rules. + * Simplifies a string according to indexing rules. */ -function search_keywords_split($text) { - static $last = null; - static $lastsplit = null; - - if ($last == $text) { - return $lastsplit; - } - +function search_simplify($text) { // Decode entities to UTF-8 $text = decode_entities($text); // Call an external processor for word handling. search_preprocess($text); + // Baseline CJK handling + $text = preg_replace_callback('/['. PREG_CLASS_CJK .']+/u', 'search_expand_cjk', $text); + // To improve searching for numerical data such as dates, IP addresses // or version numbers, we consider a group of numerical characters // separated only by punctuation characters to be one piece. @@ -260,9 +268,45 @@ // marks, spacers, etc, to be a word boundary. $text = preg_replace('/['. PREG_CLASS_SEARCH_EXCLUDE . ']+/u', ' ', $text); + return $text; +} + +/** + * Basic CJK tokenizer. Simply splits a string into consecutive, overlapping + * pairs of characters. + */ +function search_expand_cjk($matches) { + $tokens = ' '; + // Split off first character + $last = drupal_substr($matches[0], 0, 1); + $str = substr($matches[0], strlen($last)); + // Begin loop + $l = drupal_strlen($str); + for ($i = 0; $i < $l; ++$i) { + // Grab next character + $current = drupal_substr($str, 0, 1); + $str = substr($str, strlen($last)); + $tokens .= $last . $current .' '; + $last = $current; + } + return $tokens; +} + +/** + * Splits a string into tokens for indexing. + */ +function search_index_split($text) { + static $last = null; + static $lastsplit = null; + + if ($last == $text) { + return $lastsplit; + } + // Process words + $text = search_simplify($text); $words = explode(' ', $text); - array_walk($words, '_search_keywords_truncate'); + array_walk($words, '_search_index_truncate'); // Save last keyword result $last = $text; @@ -272,28 +316,13 @@ } /** - * Helper function for array_walk in search_keywords_split. + * Helper function for array_walk in search_index_split. */ -function _search_keywords_truncate(&$text) { +function _search_index_truncate(&$text) { $text = truncate_utf8($text, 50); } /** - * Loosens up a set of search keywords by adding wildcards, if possible. - * - * @param $text - * The keywords as entered by the user. - * @return - * If more wildcards can be added, the adjusted keywords are returned. - * If the query is already as loose as possible, NULL is returned. - */ -function search_keywords_variation($text) { - $text = trim($text); - $new = preg_replace('/\*+/', '*', '*'. implode('* *', explode(' ', trim($text))) .'*'); - return ($new != $text) ? $new : NULL; -} - -/** * Invokes hook_search_preprocess() in modules. */ function search_preprocess(&$text) { @@ -302,7 +331,6 @@ } } - /** * Update the full-text search index for a particular item. * @@ -318,8 +346,9 @@ * @ingroup search */ function search_index($sid, $type, $text) { - $minimum_word_size = variable_get('minimum_word_size', 3); + $minimum_word_size = variable_get('minimum_word_size', 5); + // Link matching global $base_url; $node_regexp = '!href=[\'"]?(?:'. preg_quote($base_url) .'/)?(?:\?q=)?([^\'">]+)[\'">]!i'; @@ -350,6 +379,7 @@ $tag = false; // Odd/even counter. Tag or no tag. $link = false; // State variable for link analyser $score = 1; // Starting score per word + $accum = ' '; // Accumulator for cleaned up data $results = array(0 => array()); @@ -375,7 +405,11 @@ if (preg_match('!(?:node|book)/(?:view/)?([0-9]+)!i', $path, $match)) { $linknid = $match[1]; if ($linknid > 0) { - $link = true; + // Note: ignore links to uncachable nodes to avoid redirect bugs. + $format = db_result(db_query('SELECT format FROM {node} WHERE nid = %d', $linknid)); + if (filter_format_allowcache($format)) { + $link = true; + } } } } @@ -386,11 +420,13 @@ else { // Note: use of PREG_SPLIT_DELIM_CAPTURE above will introduce empty values if ($value != '') { - $words = search_keywords_split($value); + $words = search_index_split($value); foreach ($words as $word) { + $word = drupal_strtolower($word); + // Add word to accumulator + $accum .= $word .' '; // Check wordlength if (drupal_strlen($word) >= $minimum_word_size) { - $word = drupal_strtolower($word); if ($link) { if (!isset($results[$linknid])) { $results[$linknid] = array(); @@ -409,6 +445,9 @@ search_wipe($sid, $type); + // Insert cleaned up data into dataset + db_query("INSERT INTO {search_dataset} (sid, type, data) VALUES (%d, '%s', '%s')", $sid, $type, $accum); + // Insert results into search index foreach ($results[0] as $word => $score) { db_query("INSERT INTO {search_index} (word, sid, type, score) VALUES ('%s', %d, '%s', %d)", $word, $sid, $type, $score); @@ -426,6 +465,147 @@ } /** + * Extract a module-specific search option from a search query. e.g. 'type:book' + */ +function search_query_extract($keys, $option) { + if (preg_match('/(^| )'. $option .':([^ ]*)( |$)/i', $keys, $matches)) { + return $matches[2]; + } +} + +/** + * Return a query with the given module-specific search option inserted in. + * e.g. 'type:book'. + */ +function search_query_insert($keys, $option, $value = '') { + if (search_query_extract($keys, $option)) { + $keys = trim(preg_replace('/(^| )'. $option .':[^ ]*/i', '', $keys)); + } + if ($value != '') { + $keys .= ' '. $option .':'. $value; + } + return $keys; +} + +/** + * Parse a search query into SQL conditions. + * + * We build a query that matches the dataset bodies + */ +function search_parse_query($text) { + $keys = array('positive' => array(), 'negative' => array()); + + // Tokenize query string + preg_match_all('/ (-?)("[^"]+"|[^" ]+)/i', ' '. $text, $matches, PREG_SET_ORDER); + + if (count($matches) < 1) { + return NULL; + } + + // Classify tokens + $or = false; + foreach ($matches as $match) { + // Strip off quotes + if ($match[2]{0} == '"') { + $match[2] = substr($match[2], 1, -1); + } + // Simplify keyword according to indexing rules + $match[2] = search_simplify($match[2]); + // Negative matches + if ($match[1] == '-') { + $keys['negative'][] = $match[2]; + } + // OR operator: instead of a single keyword, we store an array of all + // OR'd keywords. + elseif ($match[2] == 'OR' && count($keys['positive'])) { + $keys['positive'][] = array(array_pop($keys['positive'])); + $or = true; + continue; + } + // Plain keyword + else { + if ($or) { + $keys['positive'][count($keys['positive']) - 1][] = $match[2]; + } + else { + $keys['positive'][] = $match[2]; + } + } + $or = false; + } + + // Convert keywords into SQL statements. + $scorewords = array(); + $query = array(); + $query2 = array(); + $arguments = array(); + $arguments2 = array(); + foreach ($keys['positive'] as $key) { + if (is_array($key) && count($key)) { + $queryor = array(); + foreach ($key as $or) { + $q = _search_parse_query($or, $scorewords); + if ($q) { + $queryor[] = $q; + $arguments[] = $or; + } + } + if (count($queryor)) { + $query[] = '('. implode(' OR ', $queryor) .')'; + } + } + else { + $q = _search_parse_query($key, $scorewords); + if ($q) { + $query[] = $q; + $arguments[] = $key; + } + } + } + foreach ($keys['negative'] as $key) { + $q = _search_parse_query($key, $scorewords, true); + if ($q) { + $query[] = $q; + $arguments[] = $key; + } + } + // We separate word-index conditions because they are not needed in the + // counting query. + if (count($scorewords) == 0) { + $query2[] = "i.word = ''"; + } + else foreach ($scorewords as $word) { + $query2[] = "i.word = '%s'"; + $arguments2[] = $word; + } + $query = implode(' AND ', $query); + $query2 = implode(' OR ', $query2); + return array($query, $arguments, $query2, $arguments2); +} + +/** + * Helper function for search_parse_query(); + */ +function _search_parse_query(&$word, &$scores, $not = false) { + // Check word/phrase + if (drupal_strlen($word) < variable_get('remove_short', 3)) { + return ''; + } + $word = drupal_strtolower($word); + // Determine the scorewords of this word/phrase + if (!$not) { + $split = search_index_split($word); + foreach ($split as $s) { + if (drupal_strlen($s) >= variable_get('minimum_word_size', 5)) { + $scores[] = $s; + } + } + } + // Return matching snippet + return "d.data ". ($not ? 'NOT ' : '') ."LIKE '%% %s %%'"; +} + +/** * Do a query on the full-text search index for a word or words. * * This function is normally only called by each module that support the @@ -434,12 +614,11 @@ * The final query is an SQL select on the search_index table. As a guide for * writing the optional extra SQL fragments (see below), use this query: * - * SELECT i.type, i.sid, i.word, SUM(i.score/t.count) AS score - * FROM {search_index} i - * $join INNER JOIN {search_total} t ON i.word = t.word - * WHERE $where AND (i.word = '...' OR ...) - * GROUP BY i.type, i.sid - * ORDER BY score DESC"; + * SELECT i.type, i.sid, SUM(i.score*t.count) AS score FROM {search_index} i + * INNER JOIN {search_total} t ON i.word = t.word + * $join + * WHERE $conditions + * GROUP BY i.type, i.sid ORDER BY score DESC * * @param $keywords * A search string as entered by the user. @@ -449,83 +628,50 @@ * * @param $join * (optional) A string to be inserted into the JOIN part of the SQL query. - * For example "INNER JOIN {node} n ON n.nid = i.sid". + * For example "INNER JOIN {node} n ON n.nid = d.sid". * * @param $where * (optional) A string to be inserted into the WHERE part of the SQL query. - * For example "(n.status > 0)". + * For example "(n.status > %d)". * - * @param $variation - * Used internally. Must not be specified. + * @param $arguments + * (optional) SQL arguments belonging to the $where parameter. * * @return * An array of SIDs for the search results. * * @ingroup search */ -function do_search($keywords, $type, $join = '', $where = '1', $variation = true) { - // Note, we replace the wildcards with U+FFFD (Replacement character) to pass - // through the keyword extractor. Multiple wildcards are collapsed into one. - $keys = preg_replace('!\*+!', '�', $keywords); +function do_search($keywords, $type, $join = '', $where = '1', $arguments = array()) { // Split into words - $keys = search_keywords_split($keys); - - $words = array(); - $arguments = array(); - $refused = array(); - // Build WHERE clause - foreach ($keys as $word) { - if (drupal_strlen($word) < variable_get('remove_short', 3)) { - if ($word != '') { - $refused[] = str_replace('�', '*', $word); - } - continue; - } - if (strpos($word, '�') !== false) { - $words[] = "i.word LIKE '%s'"; - $arguments[] = str_replace('�', '%', drupal_strtolower($word)); - } - else { - $words[] = "i.word = '%s'"; - $arguments[] = drupal_strtolower($word); - } - } - // Tell the user which words were excluded - if (count($refused) && $variation) { - $message = format_plural(count($refused), - 'The word %words was not included because it is too short.', - 'The words %words were not included because they were too short.'); - drupal_set_message(strtr($message, array('%words' => theme('placeholder', implode(', ', $refused))))); - } + $query = search_parse_query($keywords); - if (count($words) == 0) { + if ($query === NULL || $query[0] == '' || $query[2] == '') { return array(); } - $conditions = $where .' AND ('. implode(' OR ', $words) .')'; - // Get result count (for pager) - $count = db_num_rows(db_query("SELECT DISTINCT i.sid, i.type FROM {search_index} i $join WHERE $conditions", $arguments)); - if ($count == 0) { - // Try out a looser search query if nothing was found. - if ($variation && $loose = search_keywords_variation($keywords)) { - return do_search($loose, $type, $join, $where, false); - } - else { - return array(); - } + // First pass: select all possible matching sids, doing a simple index-based OR matching on the keywords. + // Note: only search_index restrictions. + $conditions = $where .' AND ('. $query[2] .')'; + $arguments = array_merge($arguments, $query[3]); + $result = db_query_temporary("SELECT i.type, i.sid, SUM(i.score*t.count) AS score FROM {search_index} i INNER JOIN {search_total} t ON i.word = t.word $join WHERE $conditions GROUP BY i.type, i.sid ORDER BY score DESC", $arguments, 'temp_search_sids'); + + // Second pass: only keep items that match the complicated keywords conditions (phrase search, negative keywords, ...) + $conditions = '('. $query[0] .')'; + $arguments = $query[1]; + $result = db_query_temporary("SELECT i.type, i.sid, i.score FROM temp_search_sids i INNER JOIN {search_dataset} d ON i.sid = d.sid AND i.type = d.type WHERE $conditions", $arguments, 'temp_search_results'); + if (($count = db_result(db_query('SELECT COUNT(*) FROM temp_search_results'))) == 0) { + return array(); } $count_query = "SELECT $count"; - // Do pager query - $query = "SELECT i.type, i.sid, SUM(i.score/t.count) AS score FROM {search_index} i $join INNER JOIN {search_total} t ON i.word = t.word WHERE $conditions GROUP BY i.type, i.sid ORDER BY score DESC"; - $result = pager_query($query, 15, 0, $count_query, $arguments); - + // Do actual search query + $result = pager_query("SELECT sid, type, score FROM temp_search_results", 10, 0, $count_query, $arguments); $results = array(); while ($item = db_fetch_object($result)) { $results[] = $item->sid; } - return $results; } @@ -548,11 +694,12 @@ // Search form submits with POST but redirects to GET. This way we can keep // the search query URL clean as a whistle: // search/type/keyword+keyword - if ($_POST['edit']['keys']) { + if (isset($_POST['op'])) { if ($type == '') { $type = 'node'; } - drupal_goto('search/'. urlencode($type) .'/'. urlencode($_POST['edit']['keys'])); + $keys = module_invoke($type, 'search', 'post', $_POST['edit']['keys']); + drupal_goto('search/'. urlencode($type) .'/'. urlencode(is_null($keys) ? $_POST['edit']['keys'] : $keys)); } else if ($type == '') { // Note: search/node can not be a default tab because it would take on the @@ -658,6 +805,7 @@ $box .= form_submit(t('Search')); $box .= ''; $output .= form_item($prompt, $box); + $output .= module_invoke($type, 'search', 'form', $keys); $output .= ''; return form($output, 'post', $action); @@ -691,8 +839,7 @@ * Used for formatting search results. * * @param $keys - * A string containing keywords. They are split into words using the same - * rules as search indexing. + * A string containing a search query. * * @param $text * The text to extract fragments from. @@ -701,7 +848,11 @@ * A string containing HTML for the excerpt. */ function search_excerpt($keys, $text) { - $keys = search_keywords_split($keys); + // Extract positive keywords and phrases + preg_match_all('/ ("([^"]+)"|(?!OR)([^" ]+))/i', ' '. $keys, $matches); + $keys = array_merge($matches[2], $matches[3]); + + // Prepare text $text = strip_tags(str_replace(array('<', '>'), array(' <', '> '), $text)); array_walk($keys, '_search_excerpt_replace'); $workkeys = $keys;