diff --git a/includes/callback_attachments_settings.inc b/includes/callback_attachments_settings.inc index 147a1ec..4645aa0 100644 --- a/includes/callback_attachments_settings.inc +++ b/includes/callback_attachments_settings.inc @@ -7,6 +7,9 @@ class SearchApiAttachmentsAlterSettings extends SearchApiAbstractAlterCallback { + // Cache table name. + const CACHE_TABLE = 'cache_search_api_attachments'; + public function alterItems(array &$items) { $exclude = array(); @@ -31,6 +34,7 @@ class SearchApiAttachmentsAlterSettings extends SearchApiAbstractAlterCallback { if (isset($item->$name)) { foreach ($item->$name as $value) { foreach ($value as $file) { + $file = (array)file_load($file['fid']); if (!in_array($file['filemime'], $exclude) && !self::is_temporary($file)) { $attachments = 'attachments_' . $name; $item->$attachments .= self::getFileContent($file) . ' '; @@ -89,20 +93,53 @@ class SearchApiAttachmentsAlterSettings extends SearchApiAbstractAlterCallback { } protected function getFileContent($file) { - if ($file['filemime'] == 'text/plain' || $file['filemime'] == 'text/x-diff') { - $ret = self::extract_simple($file); + $extraction = FALSE; + // Let's make the variable consistent. + $file = (array)$file; + + // Before running the (performance-intensive) extraction process, check + // if we already have a cached copy of the extracted data. + if (isset($file['fid'])) { + // Load cached extraction based off file ID. + $cid = 'cached_extraction_' . ':' . $file['fid']; + $cached_extraction = cache_get($cid, self::CACHE_TABLE); + + // If we have a cache hit, there really is no need to continue. + if (!empty($cached_extraction->data)) { + return $cached_extraction->data; + } } - elseif (in_array($file['filemime'], array('image/jpeg', 'image/jpg', 'image/tiff'))) { - $ret = self::extract_exif($file); + + if (file_exists($file['uri'])) { + if ($file['filemime'] == 'text/plain' || $file['filemime'] == 'text/x-diff') { + $extraction = self::extract_simple($file); + } + elseif (in_array($file['filemime'], array('image/jpeg', 'image/jpg', 'image/tiff'))) { + $extraction = self::extract_exif($file); + } + else { + $extraction_method = variable_get('search_api_attachments_extract_using', 'tika'); + // Send the extraction request to the right place depending on the + // current setting. + if ($extraction_method == 'tika') { + $extraction = self::extract_tika($file); + } + else { + $extraction = self::extract_solr($file); + } + } } - else { - $ret = self::extract_tika($file); + + // If we have actual extracted data, write it to the cache. + if ($extraction !== FALSE && isset($cid)) { + cache_set($cid, $extraction, self::CACHE_TABLE); } - return $ret; + + return $extraction; } protected function extract_simple($file) { - $text = file_get_contents(file_create_url($file['uri'])); + $text = file_get_contents(self::get_realpath($file)); $text = iconv("UTF-8", "UTF-8//IGNORE", $text); $text = filter_xss(str_replace(array('<', '>'), array(' <', '> '), $text), array()); $text = htmlspecialchars(html_entity_decode($text, ENT_NOQUOTES, 'UTF-8'), ENT_NOQUOTES, 'UTF-8'); @@ -112,7 +149,7 @@ class SearchApiAttachmentsAlterSettings extends SearchApiAbstractAlterCallback { protected function extract_exif($file) { $ret = ''; - $size = getimagesize(file_create_url($file['uri']), $info); + $size = getimagesize(self::get_realpath($file), $info); if (isset($info['APP13'])) { $iptc = iptcparse($info['APP13']); foreach ($iptc as $key => $value) { @@ -125,7 +162,7 @@ class SearchApiAttachmentsAlterSettings extends SearchApiAbstractAlterCallback { } protected function extract_tika($file) { - $filepath = file_create_url($file['uri']); + $filepath = self::get_realpath($file); $tika_path = realpath(variable_get('search_api_attachments_tika_path', '')); $tika = realpath($tika_path . '/' . variable_get('search_api_attachments_tika_jar', 'tika-app-0.8.jar')); if (!($tika) || !is_file($tika)) { @@ -141,7 +178,84 @@ class SearchApiAttachmentsAlterSettings extends SearchApiAbstractAlterCallback { } return shell_exec($cmd); } - + + /** + * Extract data using Solr (via the ExtractingRequestHandler) or using the + * remote Tika servlet. + * @see http://wiki.apache.org/solr/ExtractingRequestHandler. + * @see http://wiki.apache.org/tika/TikaJAXRS. + */ + protected function extract_solr($file) { + $extraction = FALSE; + + $filepath = self::get_realpath($file); + try { + $filename = basename($filepath); + + // Server name is stored in the index. + $server_name = $this->index->server; + + $server = search_api_server_load($server_name, TRUE); + + // Make sure this is a solr server. + if ($server->class != 'search_api_solr_service') { + throw new SearchApiException(t('Server %server is not a Solr server, unable to extract file.', array( + '%server' => $server_name, + ))); + } + + // Open a connection to the server. + $solr_connection = $server->getSolrConnection(); + + // Path for our servlet request. + $servlet_path = '/update/extract'; + + // Parameters for the extraction request. + $params = array( + 'extractOnly' => 'true', + 'resource.name' => $filename, + // Matches the -t command for the tika CLI app. + 'extractFormat' => 'text', + 'wt' => 'json', + ); + + // Heavily inspired by apachesolr_file. + // @see apachesolr_file_extract(). + // Construct a multi-part form-data POST body in $data. + $boundary = '--' . md5(uniqid(REQUEST_TIME)); + $data = "--{$boundary}\r\n"; + // The 'filename' used here becomes the property name in the response. + $data .= 'Content-Disposition: form-data; name="file"; filename="extracted"'; + $data .= "\r\nContent-Type: application/octet-stream\r\n\r\n"; + $data .= file_get_contents($filepath); + $data .= "\r\n--{$boundary}--\r\n"; + $headers = array('Content-Type' => 'multipart/form-data; boundary=' . $boundary); + $options = array( + 'method' => 'POST', + 'headers' => $headers, + 'data' => $data, + ); + + // Make a servlet request using the solr connection. + $response = $solr_connection->makeServletRequest($servlet_path, $params, $options); + + // If we have an extracted response, all is well. + if (isset($response->extracted)) { + $extraction = $response->extracted; + } + } + catch (Exception $e) { + // Log the exception to watchdog. Exceptions from Solr may be transient, + // or indicate a problem with a specific file. + watchdog('search_api_attachments', 'Exception occurred sending %filepath to Solr.', array( + '%filepath' => $file['uri'], + )); + watchdog_exception('search_api_attachments', $e); + } + + return $extraction; + } + protected function is_temporary($file) { $needle = 'temporary://'; $haystack = $file['uri']; @@ -150,4 +264,15 @@ class SearchApiAttachmentsAlterSettings extends SearchApiAbstractAlterCallback { return $result; } + protected function get_realpath($file) { + $wrapper = file_stream_wrapper_get_instance_by_uri($file['uri']); + $scheme = file_uri_scheme($file['uri']); + $local_wrappers = file_get_stream_wrappers(STREAM_WRAPPERS_LOCAL); + if (in_array($scheme, array_keys($local_wrappers))) { + return $wrapper->realpath(); + } + else { + return $wrapper->getExternalUrl(); + } + } } diff --git a/search_api_attachments.admin.inc b/search_api_attachments.admin.inc index f84e0b2..7dcb603 100644 --- a/search_api_attachments.admin.inc +++ b/search_api_attachments.admin.inc @@ -9,15 +9,30 @@ /** * Page callback to show the settings for the attachments. */ -function search_api_attachments_settings_form() { - $form = array(); - $form['search_api_attachments_tika_path'] = array( +function search_api_attachments_settings_form($form, &$form_state) { + $form['search_api_attachments_extract_using'] = array( + '#type' => 'radios', + '#title' => t('Extraction method'), + '#options' => array( + 'tika' => t('Tika (local java application)'), + 'solr' => t('Solr (remote server)'), + ), + '#description' => t('Choose extraction method, remote or local (extraction will be faster if run locally using tika).'), + '#default_value' => variable_get('search_api_attachments_extract_using', 'tika'), + ); + $form['search_api_attachments_tika_settings'] = array( + '#type' => 'fieldset', + '#title' => t('Tika settings'), + '#description' => t('Required if using the "Tika" extraction method (above).'), + '#collapsible' => TRUE, + ); + $form['search_api_attachments_tika_settings']['search_api_attachments_tika_path'] = array( '#type' => 'textfield', '#title' => t('Tika directory path:'), '#default_value' => variable_get('search_api_attachments_tika_path', ''), '#description' => t('The full path to tika directory. All library jars must be in the same directory. E.g. /var/apache-tika-1.0/'), ); - $form['search_api_attachments_tika_jar'] = array( + $form['search_api_attachments_tika_settings']['search_api_attachments_tika_jar'] = array( '#type' => 'textfield', '#title' => t('Tika jar file:'), '#default_value' => variable_get('search_api_attachments_tika_jar', 'tika-app-1.0.jar'), @@ -30,11 +45,14 @@ function search_api_attachments_settings_form() { * Validation handler for the settings form. */ function search_api_attachments_settings_form_validate($form, &$form_state) { - if (empty($form_state['values']['search_api_attachments_tika_jar'])) { - form_set_error('search_api_attachments_tika_jar', t('Tika jar is mandatory.')); - } - $path = realpath($form_state['values']['search_api_attachments_tika_path']); - if (!file_exists($path . '/' . $form_state['values']['search_api_attachments_tika_jar'])) { - form_set_error('search_api_attachments_tika_path', t('Tika jar file not found at this path.')); + // We only need to validate the tika path if we're using local extraction. + if ($form_state['values']['search_api_attachments_extract_using'] == 'tika') { + if (empty($form_state['values']['search_api_attachments_tika_jar'])) { + form_set_error('search_api_attachments_tika_jar', t('Tika jar is mandatory.')); + } + $path = realpath($form_state['values']['search_api_attachments_tika_path']); + if (!file_exists($path . '/' . $form_state['values']['search_api_attachments_tika_jar'])) { + form_set_error('search_api_attachments_tika_path', t('Tika jar file not found at this path.')); + } } } diff --git a/search_api_attachments.install b/search_api_attachments.install index 7e71d51..9921038 100644 --- a/search_api_attachments.install +++ b/search_api_attachments.install @@ -13,17 +13,20 @@ function search_api_attachments_requirements($phase) { $requirements = array(); $t = get_t(); if ($phase == 'runtime') { - $temp = tempnam(file_directory_temp(), 'asa'); - exec('java -version > ' . $temp . ' 2>&1'); - $stderror = file_get_contents($temp); - $found = preg_match('/Runtime Environment/', $stderror); + // We only need to test Java if we're using a local tika extractor. + if (variable_get('search_api_attachments_extract_using', 'tika') == 'tika') { + $temp = tempnam(file_directory_temp(), 'asa'); + exec('java -version > ' . $temp . ' 2>&1'); + $stderror = file_get_contents($temp); + $found = preg_match('/Runtime Environment/', $stderror); - if (!$found) { - $requirements['search_api_attachments_java'] = array( - 'title' => $t('Java executable not found'), - 'description' => $t('Could not execute a java command. You may need to set the path of the correct java executable as the variable \'apachesolr_attachments_java\' in settings.php.'), - 'severity' => REQUIREMENT_ERROR, - ); + if (!$found) { + $requirements['search_api_attachments_java'] = array( + 'title' => $t('Java executable not found'), + 'description' => $t('Could not execute a java command. You may need to set the path of the correct java executable as the variable \'apachesolr_attachments_java\' in settings.php.'), + 'severity' => REQUIREMENT_ERROR, + ); + } } } return $requirements; @@ -35,4 +38,24 @@ function search_api_attachments_requirements($phase) { function search_api_attachments_uninstall() { variable_del('search_api_attachments_tika_path'); variable_del('search_api_attachments_tika_jar'); -} \ No newline at end of file +} + +/** + * Implements hook_schema(). + */ +function search_api_attachments_schema() { + // Create a standard Drupal cache table. + // We'll be using this store file extractions. + $schema['cache_search_api_attachments'] = drupal_get_schema_unprocessed('system', 'cache'); + return $schema; +} + +/** + * Add a cache table to store file extractions. + */ +function search_api_attachments_update_7101() { + if (!db_table_exists('cache_search_api_attachments')) { + $schema = search_api_attachments_schema(); + db_create_table('cache_search_api_attachments', $schema['cache_search_api_attachments']); + } +} diff --git a/search_api_attachments.module b/search_api_attachments.module index 4624d6f..ed762b0 100644 --- a/search_api_attachments.module +++ b/search_api_attachments.module @@ -41,3 +41,25 @@ function search_api_attachments_default_excluded() { $default = array('aif', 'art', 'avi', 'bmp', 'gif', 'ico', 'mov', 'oga', 'ogv', 'png', 'psd', 'ra', 'ram', 'rgb'); return $default; } + +/** + * Implements hook_file_update(). + */ +function search_api_attachments_file_update($file) { + // If there is cached extracted data for this file, delete it so it can be + // re-extracted. + $file = (array)$file; + $cid = 'cached_extraction_' . ':' . $file['fid']; + cache_clear_all($cid, 'cache_search_api_attachments'); +} + +/** + * Implements hook_file_delete(). + */ +function search_api_attachments_file_delete($file) { + // If there is cached extracted data for this file, delete it. + // We don't need it anymore. + $file = (array)$file; + $cid = 'cached_extraction_' . ':' . $file['fid']; + cache_clear_all($cid, 'cache_search_api_attachments'); +}