diff --git a/README.txt b/README.txt index 4832c19..2c346f1 100644 --- a/README.txt +++ b/README.txt @@ -14,10 +14,19 @@ REQUIREMENTS Requires the ability to run java on your server and an installation of the Apache Tika library +MODULE INSTALLATION +------------------- +Copy search_api_attachments into your modules folder -INSTALLATION ------------- +Install the search_api_attachments module in your Drupal site + +Go to the configuration: admin/config/search/search_api/attachments + +Choose an extraction method and follow the instructions under the respective heading below. + +EXTRACTION CONFIGURATION (Tika) +------------------------------- On Ubuntu 10.10 Install java @@ -26,11 +35,29 @@ Install java Download Apache Tika library: http://tika.apache.org/download.html > wget http://mir2.ovh.net/ftp.apache.org/dist/tika/tika-app-1.4.jar -Copy search_api_attachments into your modules folder +Enter the full path on your server where you downloaded the jar e.g. /var/apache-tika/ +and the name of the jar file e.g. tika-app-1.4.jar -Install the search_api_attachments module in your Drupal site +EXTRACTION CONFIGURATION (Solr) +------------------------------- +This requires 1.0-RC5 or newer of the 'Search API Solr' module (namely for issue #1986284) and the Solr config files that come with it. -Go to the configuration: admin/config/search/search_api/attachments +If you're just using the 'example' application with the built-in Jetty server, you can skip straight to editing your solrconfig.xml file. Libraries should automatically be loaded. + +Before continuing, download and extract the Solr package for your particular release, if you don't have it already, e.g. +> wget http://apache.mirror.serversaustralia.com.au/lucene/solr/3.6.2/apache-solr-3.6.2.tgz +> tar xvzf apache-solr-3.6.2.tgz + +In the below instructions, replace '$SOLR_HOME' with your Solr home directory. + +From the Solr package, copy the 'dist/' and 'lib/' directories into your solr home directory, e.g.: +> cp -R apache-solr-3.6.2/dist $SOLR_HOME/ +> cp -R apache-solr-3.6.2/contrib/extraction/lib $SOLR_HOME/ + +Then edit the 'solrconfig.xml' file: +> nano $SOLR_HOME/conf/solrconfig.xml + +And add the following line: + -Enter the full path on your server where you downloaded the jar e.g. /var/apache-tika/ -and the name of the jar file e.g. tika-app-1.4.jar \ No newline at end of file +Then restart Tomcat/Jetty. diff --git a/includes/callback_attachments_settings.inc b/includes/callback_attachments_settings.inc index 6698d36..6da7266 100644 --- a/includes/callback_attachments_settings.inc +++ b/includes/callback_attachments_settings.inc @@ -7,6 +7,9 @@ class SearchApiAttachmentsAlterSettings extends SearchApiAbstractAlterCallback { + // Cache table name. + const CACHE_TABLE = 'cache_search_api_attachments'; + public function alterItems(array &$items) { $exclude = array(); @@ -31,6 +34,7 @@ class SearchApiAttachmentsAlterSettings extends SearchApiAbstractAlterCallback { if (isset($item->$name)) { foreach ($item->$name as $value) { foreach ($value as $file) { + $file = (array)file_load($file['fid']); if (!in_array($file['filemime'], $exclude) && !self::is_temporary($file)) { $attachments = 'attachments_' . $name; $item->$attachments .= self::getFileContent($file) . ' '; @@ -89,16 +93,40 @@ class SearchApiAttachmentsAlterSettings extends SearchApiAbstractAlterCallback { } protected function getFileContent($file) { - $ret = FALSE; + $extraction = FALSE; + // Let's make the variable consistent. + $file = (array)$file; + + // Before running the (performance-intensive) extraction process, check + // if we already have a cached copy of the extracted data. + if (isset($file['fid'])) { + // Load cached extraction based off file ID. + $cid = 'cached_extraction_' . ':' . $file['fid']; + $cached_extraction = cache_get($cid, self::CACHE_TABLE); + + // If we have a cache hit, there really is no need to continue. + if (!empty($cached_extraction->data)) { + return $cached_extraction->data; + } + } + if (file_exists($file['uri'])) { if ($file['filemime'] == 'text/plain' || $file['filemime'] == 'text/x-diff') { - $ret = self::extract_simple($file); + $extraction = self::extract_simple($file); } elseif (in_array($file['filemime'], array('image/jpeg', 'image/jpg', 'image/tiff'))) { - $ret = self::extract_exif($file); + $extraction = self::extract_exif($file); } else { - $ret = self::extract_tika($file); + $extraction_method = variable_get('search_api_attachments_extract_using', 'tika'); + // Send the extraction request to the right place depending on the + // current setting. + if ($extraction_method == 'tika') { + $extraction = self::extract_tika($file); + } + else { + $extraction = self::extract_solr($file); + } } } else { @@ -106,7 +134,12 @@ class SearchApiAttachmentsAlterSettings extends SearchApiAbstractAlterCallback { watchdog('search_api_attachments', "Couldn't index %filename content because this file was missing.", array('%filename' => $file['filename'])); } - return $ret; + // If we have actual extracted data, write it to the cache. + if ($extraction !== FALSE && isset($cid)) { + cache_set($cid, $extraction, self::CACHE_TABLE); + } + + return $extraction; } protected function extract_simple($file) { @@ -158,7 +191,84 @@ class SearchApiAttachmentsAlterSettings extends SearchApiAbstractAlterCallback { setlocale(LC_CTYPE, $backup_locale); return shell_exec($cmd); } - + + /** + * Extract data using Solr (via the ExtractingRequestHandler) or using the + * remote Tika servlet. + * @see http://wiki.apache.org/solr/ExtractingRequestHandler. + * @see http://wiki.apache.org/tika/TikaJAXRS. + */ + protected function extract_solr($file) { + $extraction = FALSE; + + $filepath = self::get_realpath($file); + try { + $filename = basename($filepath); + + // Server name is stored in the index. + $server_name = $this->index->server; + + $server = search_api_server_load($server_name, TRUE); + + // Make sure this is a solr server. + if ($server->class != 'search_api_solr_service') { + throw new SearchApiException(t('Server %server is not a Solr server, unable to extract file.', array( + '%server' => $server_name, + ))); + } + + // Open a connection to the server. + $solr_connection = $server->getSolrConnection(); + + // Path for our servlet request. + $servlet_path = '/update/extract'; + + // Parameters for the extraction request. + $params = array( + 'extractOnly' => 'true', + 'resource.name' => $filename, + // Matches the -t command for the tika CLI app. + 'extractFormat' => 'text', + 'wt' => 'json', + ); + + // Heavily inspired by apachesolr_file. + // @see apachesolr_file_extract(). + // Construct a multi-part form-data POST body in $data. + $boundary = '--' . md5(uniqid(REQUEST_TIME)); + $data = "--{$boundary}\r\n"; + // The 'filename' used here becomes the property name in the response. + $data .= 'Content-Disposition: form-data; name="file"; filename="extracted"'; + $data .= "\r\nContent-Type: application/octet-stream\r\n\r\n"; + $data .= file_get_contents($filepath); + $data .= "\r\n--{$boundary}--\r\n"; + $headers = array('Content-Type' => 'multipart/form-data; boundary=' . $boundary); + $options = array( + 'method' => 'POST', + 'headers' => $headers, + 'data' => $data, + ); + + // Make a servlet request using the solr connection. + $response = $solr_connection->makeServletRequest($servlet_path, $params, $options); + + // If we have an extracted response, all is well. + if (isset($response->extracted)) { + $extraction = $response->extracted; + } + } + catch (Exception $e) { + // Log the exception to watchdog. Exceptions from Solr may be transient, + // or indicate a problem with a specific file. + watchdog('search_api_attachments', 'Exception occurred sending %filepath to Solr.', array( + '%filepath' => $file['uri'], + )); + watchdog_exception('search_api_attachments', $e); + } + + return $extraction; + } + protected function is_temporary($file) { $needle = 'temporary://'; $haystack = $file['uri']; diff --git a/search_api_attachments.admin.inc b/search_api_attachments.admin.inc index 51194c7..1dd74c4 100644 --- a/search_api_attachments.admin.inc +++ b/search_api_attachments.admin.inc @@ -9,15 +9,30 @@ /** * Page callback to show the settings for the attachments. */ -function search_api_attachments_settings_form() { - $form = array(); - $form['search_api_attachments_tika_path'] = array( +function search_api_attachments_settings_form($form, &$form_state) { + $form['search_api_attachments_extract_using'] = array( + '#type' => 'radios', + '#title' => t('Extraction method'), + '#options' => array( + 'tika' => t('Tika (local java application)'), + 'solr' => t('Solr (remote server)'), + ), + '#description' => t('Choose extraction method, remote or local (extraction will be faster if run locally using tika).'), + '#default_value' => variable_get('search_api_attachments_extract_using', 'tika'), + ); + $form['search_api_attachments_tika_settings'] = array( + '#type' => 'fieldset', + '#title' => t('Tika settings'), + '#description' => t('Required if using the "Tika" extraction method (above).'), + '#collapsible' => TRUE, + ); + $form['search_api_attachments_tika_settings']['search_api_attachments_tika_path'] = array( '#type' => 'textfield', '#title' => t('Tika directory path:'), '#default_value' => variable_get('search_api_attachments_tika_path', ''), '#description' => t('The full path to tika directory. All library jars must be in the same directory. E.g. /var/apache-tika-4.0/'), ); - $form['search_api_attachments_tika_jar'] = array( + $form['search_api_attachments_tika_settings']['search_api_attachments_tika_jar'] = array( '#type' => 'textfield', '#title' => t('Tika jar file:'), '#default_value' => variable_get('search_api_attachments_tika_jar', 'tika-app-1.4.jar'), @@ -30,11 +45,14 @@ function search_api_attachments_settings_form() { * Validation handler for the settings form. */ function search_api_attachments_settings_form_validate($form, &$form_state) { - if (empty($form_state['values']['search_api_attachments_tika_jar'])) { - form_set_error('search_api_attachments_tika_jar', t('Tika jar is mandatory.')); - } - $path = realpath($form_state['values']['search_api_attachments_tika_path']); - if (!file_exists($path . '/' . $form_state['values']['search_api_attachments_tika_jar'])) { - form_set_error('search_api_attachments_tika_path', t('Tika jar file not found at this path.')); + // We only need to validate the tika path if we're using local extraction. + if ($form_state['values']['search_api_attachments_extract_using'] == 'tika') { + if (empty($form_state['values']['search_api_attachments_tika_jar'])) { + form_set_error('search_api_attachments_tika_jar', t('Tika jar is mandatory.')); + } + $path = realpath($form_state['values']['search_api_attachments_tika_path']); + if (!file_exists($path . '/' . $form_state['values']['search_api_attachments_tika_jar'])) { + form_set_error('search_api_attachments_tika_path', t('Tika jar file not found at this path.')); + } } } diff --git a/search_api_attachments.install b/search_api_attachments.install index 2908d4f..dd7d6b9 100644 --- a/search_api_attachments.install +++ b/search_api_attachments.install @@ -13,21 +13,24 @@ function search_api_attachments_requirements($phase) { $requirements = array(); $t = get_t(); if ($phase == 'runtime') { - $temp = tempnam(file_directory_temp(), 'asa'); - $cmd = 'java -version > ' . $temp . ' 2>&1'; - if (strpos(ini_get('extension_dir'), 'MAMP/')) { - $cmd = 'export DYLD_LIBRARY_PATH=""; ' . $cmd; - } - exec($cmd); - $stderror = file_get_contents($temp); - $found = preg_match('/Runtime Environment/', $stderror); + // We only need to test Java if we're using a local tika extractor. + if (variable_get('search_api_attachments_extract_using', 'tika') == 'tika') { + $temp = tempnam(file_directory_temp(), 'asa'); + $cmd = 'java -version > ' . $temp . ' 2>&1'; + if (strpos(ini_get('extension_dir'), 'MAMP/')) { + $cmd = 'export DYLD_LIBRARY_PATH=""; ' . $cmd; + } + exec($cmd); + $stderror = file_get_contents($temp); + $found = preg_match('/Runtime Environment/', $stderror); - if (!$found) { - $requirements['search_api_attachments_java'] = array( - 'title' => $t('Java executable not found'), - 'description' => $t('Could not execute a java command. You may need to set the path of the correct java executable as the variable \'apachesolr_attachments_java\' in settings.php.'), - 'severity' => REQUIREMENT_ERROR, - ); + if (!$found) { + $requirements['search_api_attachments_java'] = array( + 'title' => $t('Java executable not found'), + 'description' => $t('Could not execute a java command. You may need to set the path of the correct java executable as the variable \'apachesolr_attachments_java\' in settings.php.'), + 'severity' => REQUIREMENT_ERROR, + ); + } } } return $requirements; @@ -39,4 +42,24 @@ function search_api_attachments_requirements($phase) { function search_api_attachments_uninstall() { variable_del('search_api_attachments_tika_path'); variable_del('search_api_attachments_tika_jar'); -} \ No newline at end of file +} + +/** + * Implements hook_schema(). + */ +function search_api_attachments_schema() { + // Create a standard Drupal cache table. + // We'll be using this store file extractions. + $schema['cache_search_api_attachments'] = drupal_get_schema_unprocessed('system', 'cache'); + return $schema; +} + +/** + * Add a cache table to store file extractions. + */ +function search_api_attachments_update_7101() { + if (!db_table_exists('cache_search_api_attachments')) { + $schema = search_api_attachments_schema(); + db_create_table('cache_search_api_attachments', $schema['cache_search_api_attachments']); + } +} diff --git a/search_api_attachments.module b/search_api_attachments.module index 4542eb9..e8d6f64 100644 --- a/search_api_attachments.module +++ b/search_api_attachments.module @@ -41,3 +41,25 @@ function search_api_attachments_default_excluded() { $default = array('aif', 'art', 'avi', 'bmp', 'gif', 'ico', 'mov', 'oga', 'ogv', 'png', 'psd', 'ra', 'ram', 'rgb'); return $default; } + +/** + * Implements hook_file_update(). + */ +function search_api_attachments_file_update($file) { + // If there is cached extracted data for this file, delete it so it can be + // re-extracted. + $file = (array)$file; + $cid = 'cached_extraction_' . ':' . $file['fid']; + cache_clear_all($cid, 'cache_search_api_attachments'); +} + +/** + * Implements hook_file_delete(). + */ +function search_api_attachments_file_delete($file) { + // If there is cached extracted data for this file, delete it. + // We don't need it anymore. + $file = (array)$file; + $cid = 'cached_extraction_' . ':' . $file['fid']; + cache_clear_all($cid, 'cache_search_api_attachments'); +}