',
+ '#type' => 'submit',
+ '#value' => t('Delete the attachments from the index'),
+ '#submit' => array('apachesolr_attachments_index_action_form_delete_submit'),
);
-
return $form;
}
/**
- * Submit function for the index action form.
+ * Submit handler for the Indexer actions form, delete button.
*/
-function apachesolr_attachments_index_action_form_submit($form, &$form_state) {
- switch ($form_state['values']['action']) {
- case 'reindex':
- $form_state['redirect'] = 'admin/config/search/apachesolr/attachments/confirm/reindex';
- break;
-
- case 'delete':
- $form_state['redirect'] = 'admin/config/search/apachesolr/attachments/confirm/delete';
- break;
+function apachesolr_attachments_index_action_form_reset_submit($form, &$form_state) {
+ $destination = array();
+ if (isset($_GET['destination'])) {
+ $destination = drupal_get_destination();
+ unset($_GET['destination']);
+ }
+ $env_id = $form_state['values']['env_id'];
+ $form_state['redirect'] = array('admin/config/search/apachesolr/attachments/confirm/clear-cache', array('query' => $destination));
+}
- case 'clear-cache':
- $form_state['redirect'] = 'admin/config/search/apachesolr/attachments/confirm/clear-cache';
- break;
+/**
+ * Submit handler for the Indexer actions form, delete button.
+ */
+function apachesolr_attachments_index_action_form_delete_submit($form, &$form_state) {
+ $destination = array();
+ if (isset($_GET['destination'])) {
+ $destination = drupal_get_destination();
+ unset($_GET['destination']);
}
+ $env_id = $form_state['values']['env_id'];
+ $form_state['redirect'] = array('admin/config/search/apachesolr/attachments/confirm/delete', array('query' => $destination));
}
/**
@@ -193,9 +165,6 @@ function apachesolr_attachments_confirm($form, $form_state, $operation) {
'#value' => $operation,
);
switch ($operation) {
- case 'reindex':
- $text = t('Are you sure you want to re-index the text of all file attachments?');
- break;
case 'delete':
$text = t('Are you sure you want to delete and re-index the text of all file attachments?');
break;
@@ -214,7 +183,7 @@ function apachesolr_attachments_confirm($form, $form_state, $operation) {
function apachesolr_attachments_confirm_submit($form, &$form_state) {
switch ($form_state['values']['operation']) {
case 'delete':
- if (apachesolr_attachments_delete_index()) {
+ if (apachesolr_attachments_delete_index() && apachesolr_attachments_solr_reindex()) {
drupal_set_message(t('File text has been deleted from the Apache Solr index. You must now
run cron until all files have been re-indexed.', array('!url' => url('admin/reports/status/run-cron', array('query' => array('destination' => 'admin/config/search/apachesolr/index'))))));
}
else {
@@ -226,12 +195,8 @@ function apachesolr_attachments_confirm_submit($form, &$form_state) {
}
}
break;
- case 'reindex':
- apachesolr_clear_last_index('apachesolr_attachments');
- drupal_set_message(t('All file attachments will be re-indexed.'));
- break;
case 'clear-cache':
- db_delete('apachesolr_attachments_files')->condition('removed', 0)->execute();
+ apachesolr_attachments_solr_reindex();
drupal_set_message(t('The local cache of extracted text has been deleted.'));
break;
}
@@ -246,8 +211,8 @@ function apachesolr_attachments_delete_index() {
$solr = apachesolr_get_solr();
$solr->deleteByQuery("entity_type:file AND hash:" . apachesolr_site_hash());
$solr->commit();
- apachesolr_index_set_last_updated(REQUEST_TIME);
- apachesolr_clear_last_index('apachesolr_attachments');
+ module_load_include('inc', 'apachesolr', 'apachesolr.index');
+ apachesolr_attachments_solr_reindex();
return TRUE;
}
catch (Exception $e) {
diff --git a/apachesolr_attachments.index.inc b/apachesolr_attachments.index.inc
index d2223a3..d9c80ff 100644
--- a/apachesolr_attachments.index.inc
+++ b/apachesolr_attachments.index.inc
@@ -6,162 +6,6 @@
*/
/**
- * Callback for apachesolr_index_nodes().
- *
- * Adds a document for each indexable file attachment for the given node ID.
- *
- * @see apachesolr_node_to_document()
- */
-function apachesolr_attachments_add_documents($node, $namespace) {
- $documents = array();
- $hash = apachesolr_site_hash();
- // Let any module exclude this node from the index.
- $build_document = TRUE;
- foreach (module_implements('apachesolr_node_exclude') as $module) {
- $exclude = module_invoke($module, 'apachesolr_node_exclude', $node, $namespace);
- if (!empty($exclude)) {
- $build_document = FALSE;
- }
- }
-
- if ($build_document) {
- // Since there is no notification for an attachment being unassociated with a
- // node (but that action will trigger it to be indexed again), we check for
- // fids that were added before but no longer present on this node.
- $fids = array();
- $result = db_query("SELECT fid FROM {apachesolr_attachments_files} WHERE nid = :nid", array(':nid' => $node->nid));
- foreach ($result as $row) {
- $fids[$row->fid] = $row->fid;
- }
-
- $files = apachesolr_attachments_get_indexable_files($node);
-
- // Find deleted files.
- $missing_fids = array_diff_key($fids, $files);
- if ($missing_fids) {
- db_update('apachesolr_attachments_files')->fields(array('removed' => 1))->condition('fid', $missing_fids, 'IN')->execute();
- }
- $new_files = array_diff_key($files, $fids);
-
- // Add new files.
- foreach ($new_files as $file) {
- db_insert('apachesolr_attachments_files')->fields(array(
- 'fid' => $file['fid'],
- 'nid' => $node->nid,
- 'removed' => 0,
- 'hash' => '',
- ))->execute();
- }
-
- foreach ($files as $file) {
- $text = apachesolr_attachments_get_attachment_text($file);
- if ($text) {
- $document = new ApacheSolrDocument();
- // A single file might be attached to multiple nodes.
- $document->id = apachesolr_document_id($file['fid'] . '-' . $node->nid, 'file');
- $document->site = url(NULL, array('absolute' => TRUE));
- $document->hash = $hash;
- $document->entity_type = 'file';
- $document->entity_id = $file['fid'];
- $document->bundle = $node->type;
- $document->bundle_name = node_type_get_name($node);
- $document->label = $file['filename'];
- $document->is_nid = $node->nid;
- $document->url = file_create_url($file['uri']);
- $document->path = file_stream_wrapper_get_instance_by_uri($file['uri'])->getDirectoryPath() . '/' . file_uri_target($file['uri']);
-
- $document->content = $file['filename'] . ' ' . apachesolr_clean_text($file['description']) . ' ' . $text;
-
- $document->ss_name = $node->name;
- // We want the name to ale be searchable for keywords.
- $document->tos_name = $node->name;
-
- // Everything else uses dynamic fields
- $document->is_uid = $node->uid;
- $document->bs_status = $node->status;
- $document->bs_sticky = $node->sticky;
- $document->bs_promote = $node->promote;
- $document->is_tnid = $node->tnid;
- $document->bs_translate = $node->translate;
- if (empty($node->language)) {
- // 'und' is the language-neutral code in Drupal 7.
- $document->ss_language = LANGUAGE_NONE;
- }
- else {
- $document->ss_language = $node->language;
- }
- $document->ds_created = apachesolr_date_iso($file['timestamp']);
- $document->ds_changed = $document->ds_created;
-
- // apachesolr_attachments-specific fields.
- $document->ss_filemime = $file['filemime'];
- $document->ss_file_node_title = apachesolr_clean_text($node->title);
- $document->ss_file_node_url = url('node/' . $node->nid, array('absolute' => TRUE));
-
- // Add taxonomy to document.
- $indexed_fields = apachesolr_entity_fields('node');
- foreach ($indexed_fields as $index_key => $field_info) {
- if ($field_info['field']['type'] == 'taxonomy_term_reference') { // Add only taxonomy.
- $field_name = $field_info['field']['field_name'];
- // See if the node has fields that can be indexed
- if (isset($node->{$field_name})) {
- // Got a field.
- $function = $field_info['indexing_callback'];
- if ($function && function_exists($function)) {
- // NOTE: This function should always return an array. One
- // node field may be indexed to multiple Solr fields.
- $fields = $function($node, $field_name, $index_key, $field_info);
- foreach ($fields as $field) {
- // It's fine to use this method also for single value fields.
- $document->setMultiValue($field['key'], $field['value']);
- }
- }
- }
- }
- }
-
- // Let modules add to the document.
- foreach (module_implements('apachesolr_update_index') as $module) {
- $function = $module . '_apachesolr_update_index';
- $function($document, $node, $namespace);
- }
- drupal_alter('apachesolr_attachments_index', $document, $node, $file, $namespace);
-
- $documents[] = $document;
- }
- else {
- watchdog('Apache Solr Attachments', 'Could not extract any indexable text from %filepath', array('%filepath' => $file['filepath']), WATCHDOG_WARNING);
- }
- }
- }
- return $documents;
-}
-
-/**
- * Return all non-excluded file attachments for a particular node
- */
-function apachesolr_attachments_get_indexable_files($node) {
- $files = array();
-
- $field_names = apachesolr_attachments_get_field_names();
- foreach ($field_names as $field_name) {
- if (!empty($node->$field_name)) {
- $field = $node->$field_name;
- list($lang, $values) = each($field);
- $files = array_merge($files, $values);
- }
- }
-
- $file_list = array();
- foreach ($files as $file) {
- if (apachesolr_attachments_allowed_mime($file['filemime'])) {
- $file_list[$file['fid']] = $file;
- }
- }
- return $file_list;
-}
-
-/**
* Checks if a file is of a MIME type that is to be excluded from the index.
*
* The MIME types of excluded files are built and cached each time the file
@@ -199,39 +43,18 @@ function apachesolr_attachments_allowed_mime($filemime) {
}
/**
- * Return all fields that are of type 'file'
- */
-function apachesolr_attachments_get_field_names() {
- $field_names = array();
- if (module_exists('file')) {
- $fields = field_info_fields();
- foreach ($fields as $field_name => $field) {
- if ($field['type'] == 'file') {
- $field_names[] = $field_name;
- }
- }
- }
- return $field_names;
-}
-
-/**
* Parse the attachment getting just the raw text.
*
* @throws Exception
*/
function apachesolr_attachments_get_attachment_text($file) {
- $filepath = drupal_realpath($file['uri']);
- // Check that we have a valid filepath.
- if (!$filepath) {
- return FALSE;
- }
- elseif (!is_file($filepath)) {
- watchdog('Apache Solr Attachments', '%filepath is not a valid file path', array('%filepath' => $file['uri']), WATCHDOG_WARNING);
- return FALSE;
+ if (!apachesolr_attachments_is_file($file)) {
+ return FALSE;
}
+ $filepath = drupal_realpath($file->uri);
// No need to use java for plain text files.
- if ($file['filemime'] == 'text/plain' || $file['filemime'] == 'text/x-diff') {
+ if ($file->filemime == 'text/plain' || $file->filemime == 'text/x-diff') {
$text = file_get_contents($filepath);
// TODO - try to detect encoding and convert to UTF-8.
// Strip bad control characters.
@@ -246,7 +69,8 @@ function apachesolr_attachments_get_attachment_text($file) {
return FALSE;
}
- $cached = db_query("SELECT * FROM {apachesolr_attachments_files} WHERE fid = :fid", array(':fid' => $file['fid']))->fetchAssoc();
+ $indexer_table = apachesolr_get_indexer_table('file');
+ $cached = db_query("SELECT * FROM {{$indexer_table}} WHERE entity_id = :entity_id", array(':entity_id' => $file->fid))->fetchAssoc();
if (!is_null($cached['body']) && ($cached['hash'] == $hash)) {
// No need to re-extract.
@@ -263,7 +87,7 @@ function apachesolr_attachments_get_attachment_text($file) {
}
catch (Exception $e) {
// Exceptions from Solr may be transient, or indicate a problem with a specific file.
- watchdog('Apache Solr Attachments', "Exception occurred sending %filepath to Solr\n!message", array('%filepath' => $file['uri'], '!message' => nl2br(check_plain($e->getMessage()))), WATCHDOG_ERROR);
+ watchdog('Apache Solr Attachments', "Exception occurred sending %filepath to Solr\n!message", array('%filepath' => $file->uri, '!message' => nl2br(check_plain($e->getMessage()))), WATCHDOG_ERROR);
return FALSE;
}
}
@@ -273,7 +97,7 @@ function apachesolr_attachments_get_attachment_text($file) {
$text = trim(apachesolr_clean_text($text));
// Save the extracted, cleaned text to the DB.
- db_update('apachesolr_attachments_files')->fields(array('hash' => $hash, 'body' => $text))->condition('fid', $file['fid'])->execute();
+ db_update('apachesolr_index_entities_file')->fields(array('hash' => $hash, 'body' => $text))->condition('entity_id', $file->fid)->execute();
return $text;
}
@@ -286,7 +110,7 @@ function apachesolr_attachments_get_attachment_text($file) {
function apachesolr_attachments_extract_using_tika($filepath) {
$tika_path = realpath(variable_get('apachesolr_attachments_tika_path', ''));
- $tika = realpath($tika_path . '/' . variable_get('apachesolr_attachments_tika_jar', 'tika-0.3.jar'));
+ $tika = realpath($tika_path . '/' . variable_get('apachesolr_attachments_tika_jar', 'tika-app-1.1.jar'));
if (!$tika || !is_file($tika)) {
throw new Exception(t('Invalid path or filename for tika application jar.'));
}
@@ -312,7 +136,7 @@ function apachesolr_attachments_extract_using_tika($filepath) {
}
/**
- * For a file path, try to extract text using Solr 1.4.
+ * For a file path, try to extract text using Solr 1.4+.
*
* @throws Exception
*/
@@ -320,7 +144,8 @@ function apachesolr_attachments_extract_using_solr($filepath) {
// Extract using Solr.
// We allow Solr to throw exceptions - they will be caught
// by apachesolr.module.
- $solr = apachesolr_get_solr();
+ $env_id = apachesolr_default_environment();
+ $solr = apachesolr_get_solr($env_id);
$filename = basename($filepath);
$params = array(
'resource.name' => $filename,
@@ -343,3 +168,64 @@ function apachesolr_attachments_extract_using_solr($filepath) {
$response = $solr->makeServletRequest(EXTRACTING_SERVLET, $params, $options);
return array($response->extracted, $response->extracted_metadata);
}
+
+/**
+ * Records that a parent entity is using a file.
+ *
+ * @param $file
+ * A file object.
+ * @param $module
+ * The name of the module using the file.
+ * @param $type
+ * The type of the object that contains the referenced file.
+ * @param $id
+ * The unique, numeric ID of the object containing the referenced file.
+ * @param $count
+ * (optional) The number of references to add to the object. Defaults to 1.
+ *
+ */
+function apachesolr_attachments_add_file_usage(stdClass $file, $parent_entity_type, $parent_entity_id) {
+ $indexer_table = apachesolr_get_indexer_table('file');
+ db_merge($indexer_table)
+ ->key(array(
+ 'entity_type' => 'file',
+ 'entity_id' => $file->fid,
+ 'parent_entity_type' => $parent_entity_type,
+ 'parent_entity_id' => $parent_entity_id,
+ ))
+ ->fields(array(
+ 'bundle' => 'file',
+ 'status' => 1,
+ 'changed' => REQUEST_TIME,
+ ))
+ ->execute();
+}
+
+/**
+ * Removes a record to indicate that an entity is no longer using a file.
+ *
+ * @param $file
+ * A file object.
+ * @param $parent_entity_type
+ * (optional) The type of the object that contains the referenced file. May
+ * be omitted if all module references to a file are being deleted.
+ * @param $parent_entity_id
+ * (optional) The unique, numeric ID of the object containing the referenced
+ * file. May be omitted if all module references to a file are being deleted.
+ *
+ */
+function apachesolr_attachments_delete_file_usage(stdClass $file, $parent_entity_type = NULL, $parent_entity_id = NULL) {
+ $indexer_table = apachesolr_get_indexer_table('file');
+ $query = db_delete($indexer_table)
+ ->condition('entity_type', 'file')
+ ->condition('entity_id', $file->fid)
+ ->condition('parent_entity_type', $parent_entity_type)
+ ->condition('parent_entity_id', $parent_entity_id);
+}
+
+function apachesolr_attachments_clean_index_table() {
+ $indexer_table = apachesolr_get_indexer_table('file');
+ db_delete($indexer_table)
+ ->condition('parent_entity_id', 0)
+ ->execute();
+}
\ No newline at end of file
diff --git a/apachesolr_attachments.info b/apachesolr_attachments.info
index 5f5b253..e890286 100644
--- a/apachesolr_attachments.info
+++ b/apachesolr_attachments.info
@@ -1,7 +1,6 @@
name = Apache Solr search attachments
description = Search file attachments with Solr
dependencies[] = apachesolr
-dependencies[] = apachesolr_search
package = Search Toolkit
core = 7.x
diff --git a/apachesolr_attachments.install b/apachesolr_attachments.install
index b8df3a9..0fb8293 100644
--- a/apachesolr_attachments.install
+++ b/apachesolr_attachments.install
@@ -1,4 +1,4 @@
- 'Stores a record of when a file property changed to determine if it needs indexing by Solr.',
- 'fields' => array(
- 'fid' => array(
- 'description' => 'The primary identifier for a file.',
- 'type' => 'int',
- 'unsigned' => TRUE,
- 'not null' => TRUE,
- ),
- 'nid' => array(
- 'description' => 'The primary identifier for the node to which the file is attached.',
- 'type' => 'int',
- 'unsigned' => TRUE,
- 'not null' => TRUE,
- 'default' => 0,
- ),
- 'removed' => array(
- 'description' => 'TRUE if the file is no longer attached to a node.',
- 'type' => 'int',
- 'unsigned' => TRUE,
- 'default' => 0,
- ),
- 'hash' => array(
- 'description' => "A hash of the file's body, to check for changes.",
- 'type' => 'varchar',
- 'length' => 255,
- 'not null' => TRUE,
- 'default' => '',
+ $types = array(
+ 'file' => 'apachesolr_index_entities_file',
+ );
+ foreach ($types as $type) {
+ $schema[$type] = array(
+ 'description' => t('Stores a record of when an entity changed to determine if it needs indexing by Solr.'),
+ 'fields' => array(
+ 'entity_type' => array(
+ 'description' => t('The type of entity.'),
+ 'type' => 'varchar',
+ 'length' => 128,
+ 'not null' => TRUE,
+ ),
+ 'entity_id' => array(
+ 'description' => t('The primary identifier for an entity.'),
+ 'type' => 'int',
+ 'unsigned' => TRUE,
+ 'not null' => TRUE,
+ ),
+ 'bundle' => array(
+ 'description' => t('The bundle to which this entity belongs.'),
+ 'type' => 'varchar',
+ 'length' => 128,
+ 'not null' => TRUE,
+ ),
+ 'status' => array(
+ 'description' => t('Boolean indicating whether the entity is visible to non-administrators (eg, published for nodes).'),
+ 'type' => 'int',
+ 'not null' => TRUE,
+ 'default' => 1,
+ ),
+ 'changed' => array(
+ 'description' => t('The Unix timestamp when an entity was changed.'),
+ 'type' => 'int',
+ 'not null' => TRUE,
+ 'default' => 0,
+ ),
+ 'parent_entity_type' => array(
+ 'description' => t('The type of entity.'),
+ 'type' => 'varchar',
+ 'length' => 128,
+ 'not null' => FALSE,
+ ),
+ 'parent_entity_id' => array(
+ 'description' => t('The type of the parent entity.'),
+ 'type' => 'int',
+ 'unsigned' => TRUE,
+ 'not null' => FALSE,
+ ),
+ 'hash' => array(
+ 'description' => "A hash of the file's body, to check for changes.",
+ 'type' => 'varchar',
+ 'length' => 255,
+ 'not null' => TRUE,
+ 'default' => '',
+ ),
+ 'body' => array(
+ 'description' => 'The cached body (extracted text) of the file, unless it is a text file.',
+ 'type' => 'text',
+ 'not null' => FALSE,
+ 'size' => 'big',
+ ),
),
- 'body' => array(
- 'description' => 'The cached body (extracted text) of the file, unless it is a text file.',
- 'type' => 'text',
- 'not null' => FALSE,
- 'size' => 'big',
+ 'indexes' => array(
+ 'changed' => array('changed', 'status'),
),
- ),
- 'indexes' => array(
- 'removed' => array('removed'),
- ),
- 'primary key' => array('fid', 'nid'),
- );
-
+ 'primary key' => array('entity_id', 'parent_entity_id', 'parent_entity_type'),
+ );
+ }
return $schema;
}
@@ -147,6 +159,107 @@ function apachesolr_attachments_update_7001() {
* Change DB index to allow files attached to multiple nodes.
*/
function apachesolr_attachments_update_7002() {
- db_query('ALTER TABLE {apachesolr_attachments_files} DROP INDEX `nid`');
- db_query('ALTER TABLE {apachesolr_attachments_files} DROP PRIMARY KEY, ADD PRIMARY KEY (`fid`, `nid`)');
+ // Removed
}
+
+
+
+/**
+ * Change DB tables to support multiple entity indexing.
+ */
+function apachesolr_attachments_update_7003() {
+ $types = array(
+ 'file' => 'apachesolr_index_entities_file',
+ );
+ foreach ($types as $type) {
+ $schema[$type] = array(
+ 'description' => t('Stores a record of when an entity changed to determine if it needs indexing by Solr.'),
+ 'fields' => array(
+ 'entity_type' => array(
+ 'description' => t('The type of entity.'),
+ 'type' => 'varchar',
+ 'length' => 128,
+ 'not null' => TRUE,
+ ),
+ 'entity_id' => array(
+ 'description' => t('The primary identifier for an entity.'),
+ 'type' => 'int',
+ 'unsigned' => TRUE,
+ 'not null' => TRUE,
+ ),
+ 'bundle' => array(
+ 'description' => t('The bundle to which this entity belongs.'),
+ 'type' => 'varchar',
+ 'length' => 128,
+ 'not null' => TRUE,
+ ),
+ 'status' => array(
+ 'description' => t('Boolean indicating whether the entity is visible to non-administrators (eg, published for nodes).'),
+ 'type' => 'int',
+ 'not null' => TRUE,
+ 'default' => 1,
+ ),
+ 'changed' => array(
+ 'description' => t('The Unix timestamp when an entity was changed.'),
+ 'type' => 'int',
+ 'not null' => TRUE,
+ 'default' => 0,
+ ),
+ 'parent_entity_type' => array(
+ 'description' => t('The type of entity.'),
+ 'type' => 'varchar',
+ 'length' => 128,
+ 'not null' => FALSE,
+ ),
+ 'parent_entity_id' => array(
+ 'description' => t('The type of the parent entity.'),
+ 'type' => 'int',
+ 'unsigned' => TRUE,
+ 'not null' => FALSE,
+ ),
+ 'hash' => array(
+ 'description' => "A hash of the file's body, to check for changes.",
+ 'type' => 'varchar',
+ 'length' => 255,
+ 'not null' => TRUE,
+ 'default' => '',
+ ),
+ 'body' => array(
+ 'description' => 'The cached body (extracted text) of the file, unless it is a text file.',
+ 'type' => 'text',
+ 'not null' => FALSE,
+ 'size' => 'big',
+ ),
+ ),
+ 'indexes' => array(
+ 'changed' => array('changed', 'status'),
+ ),
+ 'primary key' => array('entity_id'),
+ );
+ }
+ if (!db_table_exists('apachesolr_index_entities_file') && db_table_exists('apachesolr_attachments_files')) {
+ db_create_table('apachesolr_index_entities_file', $schema['file']);
+ // Move some data from apachesolr_attachments_files to apachesolr_index_entities_file
+ $select = db_select('apachesolr_attachments_files', 'af');
+ $select->join('node', 'n', 'af.nid = n.nid');
+ $select->addField('n', 'nid', 'entity_id');
+ $select->addField('n', 'type', 'bundle');
+ $select->addField('af', 'fid', 'fid');
+ $select->addField('af', 'nid', 'parent_entity_id');
+ $select->addExpression("0", 'changed');
+ $select->addExpression("'node'", 'entity_type');
+ $select->addExpression("'node'", 'parent_entity_type');
+ $return_value = db_insert('apachesolr_index_entities_file')
+ ->fields(array('entity_id', 'bundle', 'fid', 'parent_entity_id', 'changed', 'entity_type', 'parent_entity_type'))
+ ->from($select)
+ ->execute();
+ }
+
+ if (db_table_exists('apachesolr_attachments_files')) {
+ db_drop_table('apachesolr_attachments_files');
+ }
+ variable_del('apachesolr_attachments_exclude_types');
+ variable_del('apachesolr_attachments_cron_limit');
+ variable_del('apachesolr_attachments_cron_time_limit');
+ variable_del('apachesolr_attachments_cron_try');
+}
\ No newline at end of file
diff --git a/apachesolr_attachments.module b/apachesolr_attachments.module
index 58644ac..41568e3 100644
--- a/apachesolr_attachments.module
+++ b/apachesolr_attachments.module
@@ -48,161 +48,274 @@ function apachesolr_attachments_menu() {
}
/**
- * Implements hook_help().
- *
- * @see apachesolr_search_help()
+ * @file
+ * Indexer for the userhook_apachesolr_entity_info_alter entities for the Apachesolr module.
*/
-function apachesolr_attachments_help($section) {
- switch ($section) {
- case 'admin/config/search/apachesolr/index':
- if (apachesolr_environment_variable_get(apachesolr_default_environment(), 'apachesolr_read_only', APACHESOLR_READ_WRITE) == APACHESOLR_READ_ONLY) {
- return t('Operating in read-only mode; updates are disabled.');
- }
- $remaining = 0;
- $total = 0;
- // Collect the stats
- $status = apachesolr_index_status('apachesolr_attachments');
- $remaining += $status['remaining'];
- $total += $status['total'];
-
- // This message is prepended to the message produced by apachesolr_search_help().
- return t('
There @items remaining to be examined for attachments out of @total total.
', array(
- '@items' => format_plural($remaining, t('is 1 item'), t('are @count items')),
- '@total' => $total,
- ));
- }
-}
-/**
- * Implements hook_search_info().
- *
- * @see apachesolr_search_search_info()
- */
-function apachesolr_attachments_search_info() {
- // We dont want a search tab, so set to same as apachesolr_search.
- return array(
- 'title' => 'Site',
- 'path' => 'site',
- 'conditions_callback' => 'apachesolr_search_conditions',
- );
+function apachesolr_attachments_apachesolr_entity_info_alter(&$entity_info) {
+ $entity_info['file']['indexable'] = TRUE;
+ $entity_info['file']['status callback'][] = 'apachesolr_attachments_status_callback';
+ $entity_info['file']['document callback'][] = 'apachesolr_attachments_solr_document';
+ $entity_info['file']['reindex callback'] = 'apachesolr_attachments_solr_reindex';
+ $entity_info['file']['index_table'] = 'apachesolr_index_entities_file';
+ $entity_info['file']['result callback'] = 'apachesolr_attachments_file_result';
}
/**
- * Implements hook_search_reset().
+ * Builds the file-specific information for a Solr document.
*
- * @see apachesolr_search_search_reset()
+ * @param ApacheSolrDocument $document
+ * The Solr document we are building up.
+ * @param stdClass $entity
+ * The entity we are indexing.
+ * @param string $entity_type
+ * The type of entity we're dealing with.
*/
-function apachesolr_attachments_search_reset() {
- apachesolr_clear_last_index('apachesolr_attachments');
-}
+function apachesolr_attachments_solr_document(ApacheSolrDocument $document, $file, $entity_type, $env_id) {
+ module_load_include('inc', 'apachesolr_attachments', 'apachesolr_attachments.index');
+ $documents = array();
-/**
- * Implements hook_search_status().
- *
- * @see apachesolr_search_search_status()
- */
-function apachesolr_attachments_search_status() {
- // TODO: Figure out a way to know how many actual files are left to update.
- return apachesolr_index_status('apachesolr_attachments');
-}
-/**
- * Implements hook_search_execute().
- */
-function apachesolr_attachments_search_execute($keys = NULL, $conditions = NULL) {
- // We dont want a search tab. Don't do anything.
- return array();
+ $text = apachesolr_attachments_get_attachment_text($file);
+ // Text is saved in the index table. Will be used by the node indexing if
+ // available.
+ // Function should exist that updates the changed data for a node when the
+ // file was updated/extracted
+
+ // Get list of nodes that are linked to this particular file
+ // A single file might be attached to multiple nodes.
+ $parents = file_get_file_references($file, NULL, FIELD_LOAD_CURRENT);
+ $parents_list = $parents ? reset($parents) : NULL;
+ if (!empty($parents_list)) {
+ foreach ($parents_list as $parent_entity_type => $parent) {
+ foreach ($parent as $parent_entity_id => $parent_info) {
+ // load the parent entity and reset cache
+ $parent_entities = entity_load($parent_entity_type, array($parent_entity_id), NULL, TRUE);
+ $parent_entity = reset($parent_entities);
+ // Skip invalid entities
+ if (empty($parent_entity)) {
+ continue;
+ }
+
+ // Get a clone of the bare minimum document
+ $filedocument = clone $document;
+
+ //Get the callback array to add stuff to the document
+ $callbacks = apachesolr_entity_get_callback($parent_entity_type, 'document callback');
+ $build_documents = array();
+ foreach ($callbacks as $callback) {
+ // Call a type-specific callback to add stuff to the document.
+ $build_documents = array_merge($build_documents, $callback($filedocument, $parent_entity, $parent_entity_type, $env_id));
+ }
+
+ // Take the top document from the stack
+ $filedocument = reset($build_documents);
+
+ // Build our separate document and overwrite basic information
+ $filedocument->id = apachesolr_document_id($file->fid . '-' . $parent_entity_id, 'file');
+ $filedocument->url = file_create_url($file->uri);
+ $filedocument->path = file_stream_wrapper_get_instance_by_uri($file->uri)->getExternalUrl();
+
+ // Add extra info to our document
+ $filedocument->label = apachesolr_clean_text($file->filename);
+ $filedocument->content = apachesolr_clean_text($file->filename) . ' ' . $text;
+
+ $filedocument->ds_created = apachesolr_date_iso($file->timestamp);
+ $filedocument->ds_changed = $filedocument->ds_created;
+
+ $filedocument->created = apachesolr_date_iso($file->timestamp);
+ $filedocument->changed = $filedocument->created;
+
+ // Add Apachesolr Attachments specific fields.
+ $filedocument->is_parent_entity_id = $parent_entity_id;
+ $filedocument->ss_filemime = $file->filemime;
+ $filedocument->ss_filesize = $file->filesize;
+
+ // Generate the path to the node
+ $filedocument->ss_file_entity_title = apachesolr_clean_text($parent_entity->title);
+ $parent_entity_uri = entity_uri($parent_entity_type, $parent_entity);
+ $parent_entity_uri['options']['absolute'] = TRUE;
+ $filedocument->ss_file_entity_url = url($parent_entity_uri['path'], $parent_entity_uri['options']);
+ $documents[] = $filedocument;
+ }
+ }
+ }
+ return $documents;
}
/**
- * Implements hook_apachesolr_types_exclude().
+ * Reindexing callback for ApacheSolr, for file entities.
*/
-function apachesolr_attachments_apachesolr_types_exclude($namespace) {
- if ($namespace == 'apachesolr_attachments' && variable_get('apachesolr_attachments_exclude_types', 1)) {
- return apachesolr_search_apachesolr_types_exclude('apachesolr_search');
+function apachesolr_attachments_solr_reindex() {
+ module_load_include('inc', 'apachesolr_attachments', 'apachesolr_attachments.index');
+ $indexer_table = apachesolr_get_indexer_table('file');
+ $transaction = db_transaction();
+ $env_id = apachesolr_default_environment();
+ try {
+ // Clean the table
+ db_delete($indexer_table)
+ ->condition('entity_type', 'file')
+ ->execute();
+ $files = _apachesolr_attachments_get_all_files();
+ // If we do not have files, return success
+ if (empty($files)) {
+ return TRUE;
+ }
+ // Loop over all the files and add them to our indexing table
+ foreach ($files as $parent_entity_type => $parent_entities) {
+ foreach ($parent_entities as $parent_entity_id => $parent_entity_info) {
+ // Fake our file class
+ $file = new stdClass();
+ $file->fid = $parent_entity_info->file_fid;
+ apachesolr_attachments_add_file_usage($file, $parent_entity_type, $parent_entity_id);
+ }
+ }
}
+ catch (Exception $e) {
+ $transaction->rollback();
+ //drupal_set_message($e->getMessage(), 'error');
+ watchdog_exception('Apache Solr Attachments', $e);
+ return FALSE;
+ }
+ return TRUE;
}
-/**
- * Implements hook_apachesolr_document_handlers().
- *
- * @see apachesolr_search_apachesolr_document_handlers()
- */
-function apachesolr_attachments_apachesolr_document_handlers($type, $namespace) {
- if ($type == 'node' && $namespace == 'apachesolr_attachments') {
- return array('apachesolr_attachments_add_documents');
+function _apachesolr_attachments_get_all_files() {
+ $results = array();
+ $fields = field_info_field_by_ids();
+ foreach ($fields as $field_id => $field_info) {
+ if ($field_info['type'] == 'file') {
+ foreach ($field_info['bundles'] as $entity_type => $bundles) {
+ $query = new ApachesolrAttachmentsEntityFieldQuery();
+ $results = $query
+ ->entityCondition('entity_type', $entity_type)
+ ->propertyCondition('status', 1)
+ ->fieldCondition($field_info['field_name'])
+ ->addTag('apachesolr_attachments_solr_reindex')
+ ->addMetaData('field_fid_key', $field_info['field_name']. '_fid')
+ ->execute();
+ }
+ }
}
+ return $results;
}
-/**
- * Implements hook_update_index().
- *
- * Search content types and add any field that is a file type that we know how
- * to parse and any uploaded file attachments.
- *
- * @see apachesolr_search_cron()
+/*
+ * Add our file id to the result set
*/
-function apachesolr_attachments_update_index() {
- if (variable_get('apachesolr_attachments_extract_using', 'tika') == 'solr' || variable_get('apachesolr_attachments_tika_path', '')) {
- module_load_include('inc', 'apachesolr_attachments', 'apachesolr_attachments.index');
- $start = REQUEST_TIME;
- $cron_try = variable_get('apachesolr_attachments_cron_try', 20);
- $cron_limit = variable_get('apachesolr_attachments_cron_limit', 100);
- $cron_time_limit = variable_get('apachesolr_attachments_cron_time_limit', 15);
- $num_tried = 0;
- do {
- $rows = apachesolr_get_nodes_to_index('apachesolr_attachments', $cron_try);
- // Calls apachesolr_attachments_apachesolr_document_handlers() and
- // ultimately apachesolr_attachments_add_documents().
- $success = apachesolr_index_nodes($rows, 'apachesolr_attachments');
- $num_tried += $cron_try;
- } while ($success && ($num_tried < $cron_limit) && (REQUEST_TIME - $start < $cron_time_limit));
+function apachesolr_attachments_query_apachesolr_attachments_solr_reindex_alter(QueryAlterableInterface $query) {
+ // We are sure that entity_id is available. Take that table and
+ // add our fid key
+ $fields = $query->getFields();
+ $query->addField($fields['entity_id']['table'], $query->getMetaData('field_fid_key'), $query->getMetaData('field_fid_key'));
+}
+
+function apachesolr_attachments_status_callback($entity_id, $entity_type) {
+ module_load_include('inc', 'apachesolr_attachments', 'apachesolr_attachments.index');
+ // Make sure we have a boolean value.
+ // Anything different from 1 becomes zero
+ if (!$entity_id) {
+ return FALSE;
}
+
+ // load the entity and reset cache
+ $entities = entity_load($entity_type, array($entity_id), NULL, TRUE);
+ $entity = reset($entities);
+ $status = ($entity->status == 1 ? 1 : 0);
+ // Check if the mimetype is allowed
+ $status = $status & apachesolr_attachments_is_file($entity);
+ $status = $status & apachesolr_attachments_allowed_mime($entity->filemime);
+ $status = $status & apachesolr_attachments_excluded_entities($entity_id, $entity_type);
+ return $status;
}
-/**
- * Implements hook_node_update().
- *
- * If node is unpublished, mark all associated attachments as removed.
- *
- * @see apachesolr_node_delete()
- */
-function apachesolr_attachments_node_update($node) {
- if (!$node->status) {
- apachesolr_attachments_node_delete($node);
+function apachesolr_attachments_is_file($entity) {
+ if (!empty($entity->uri)) {
+ $filepath = drupal_realpath($entity->uri);
+ // Check that we have a valid filepath.
+ if (!$filepath) {
+ return FALSE;
+ }
+ elseif (!is_file($filepath)) {
+ watchdog('Apache Solr Attachments', '%filepath is not a valid file path', array('%filepath' => $entity->uri), WATCHDOG_WARNING);
+ return FALSE;
+ }
+ else {
+ return TRUE;
+ }
}
+ return FALSE;
}
/**
- * Implements hook_node_delete().
- *
- * Mark all associated attachments as removed.
- *
- * @see apachesolr_node_delete()
+ * Get a list of entities that are linked to this particular file
+ * A single file might be attached to multiple entities.
+ * Excludes a file as soon as 1 entity was excluded
+ * @param type $entity_id
+ * @param type $entity_type
+ * @return type
*/
-function apachesolr_attachments_node_delete($node) {
- // Mark attachments for later deletion if the query fails.
- if (apachesolr_attachments_remove_attachments_from_index($node->nid)) {
- db_delete('apachesolr_attachments_files')->condition('nid', $node->nid)->execute();
- }
- else {
- db_update('apachesolr_attachments_files')->fields(array('removed' => 1))->condition('nid', $node->nid)->execute();
+function apachesolr_attachments_excluded_entities($entity_id, $entity_type) {
+ $env_id = apachesolr_default_environment();
+ $status = TRUE;
+
+ $file = file_load($entity_id);
+ $parents = file_get_file_references($file, NULL, FIELD_LOAD_CURRENT);
+ $parents_list = $parents ? reset($parents) : NULL;
+
+ if (!empty($parents_list)) {
+ foreach ($parents_list as $parent_entity_type => $parent) {
+ foreach ($parent as $parent_entity_id => $parent_info) {
+ // load the parent entity and reset cache
+ $parent_entities = entity_load($parent_entity_type, array($parent_entity_id), NULL, TRUE);
+ // Take the first entity from the stack
+ $parent_entity = reset($parents_entities);
+
+ // Skip invalid entities
+ if (empty($parent_entity)) {
+ continue;
+ }
+
+ // Retrieve the parent entity id
+ list($parent_entity_id) = entity_extract_ids($parent_entity_type, $parent_entity);
+
+ // Skip indexing of files if the node was excluded by apache solr
+ $status_callbacks = apachesolr_entity_get_callback($parent_entity_type, 'status callback');
+
+ // Check status callback before sending to the index
+ foreach($status_callbacks as $status_callback) {
+ if (is_callable($status_callback)) {
+ // by placing $status in front we prevent calling any other callback
+ // after one status callback returned false
+ $status = $status && $status_callback($parent_entity_id, $parent_entity_type);
+ }
+ }
+
+ // Delete the entity from our index if the status callback returns 0
+ if ($status == FALSE) {
+ // One exclusion is enough not to consider this attachment for indexing
+ apachesolr_remove_entity($env_id, $entity_type, $entity_id);
+ return FALSE;
+ }
+ }
+ }
}
+ return $status;
}
/**
- * For a particular node ID, remove all file attachments from the Solr index.
+ * For a particular entity ID, remove all file attachments from the Solr index.
*
* @see apachesolr_delete_node_from_index()
*/
-function apachesolr_attachments_remove_attachments_from_index($nid) {
+function apachesolr_attachments_remove_attachments_from_index($parent_entity_id) {
static $failed = FALSE;
if ($failed) {
return FALSE;
}
try {
$solr = apachesolr_get_solr();
- $solr->deleteByQuery("is_nid:$nid AND entity_type:file AND hash:" . apachesolr_site_hash());
+ $solr->deleteByQuery("is_parent_entity_id:$parent_entity_id AND entity_type:file AND hash:" . apachesolr_site_hash());
$solr->commit();
return TRUE;
}
@@ -215,36 +328,6 @@ function apachesolr_attachments_remove_attachments_from_index($nid) {
}
/**
- * Implements hook_cron().
- *
- * Delete all removed attachments from the Solr store.
- *
- * @see apachesolr_nodeapi_mass_delete()
- */
-function apachesolr_attachments_cron() {
- $files = db_query("SELECT fid, nid FROM {apachesolr_attachments_files} WHERE removed = :removed", array(':removed' => 1));
-
- $ids = array();
- $fids = array();
- foreach ($files as $file) {
- $ids[] = apachesolr_document_id($file->fid . '-' . $file->nid, 'file');
- $fids[] = $file->fid;
- }
- if ($ids) {
- try {
- $solr = apachesolr_get_solr();
- $solr->deleteByMultipleIds($ids);
- $solr->commit();
- // There was no exception, so update the table.
- db_delete('apachesolr_attachments_files')->condition('fid', $fids, 'IN')->execute();
- }
- catch (Exception $e) {
- watchdog('Apache Solr Attachments', nl2br(check_plain($e->getMessage())) . ' in apachesolr_attachments_cron', NULL, WATCHDOG_ERROR);
- }
- }
-}
-
-/**
* Implements hook_apachesolr_query_alter().
*/
function apachesolr_attachments_apachesolr_query_alter(DrupalSolrQueryInterface $query) {
@@ -258,13 +341,77 @@ function apachesolr_attachments_apachesolr_query_alter(DrupalSolrQueryInterface
}
}
+/*
+ * hook_entity_OP functions need to happen to clean up the table after files
+ * were added to the index table
+ */
+function apachesolr_attachments_entity_update($entity, $type) {
+ module_load_include('inc', 'apachesolr_attachments', 'apachesolr_attachments.index');
+ apachesolr_attachments_clean_index_table();
+}
+
+function apachesolr_attachments_entity_insert($entity, $type) {
+ apachesolr_attachments_entity_update($entity, $type);
+}
+
+function apachesolr_attachments_entity_delete($entity, $type) {
+ module_load_include('inc', 'apachesolr_attachments', 'apachesolr_attachments.index');
+ apachesolr_attachments_clean_index_table();
+}
+
/**
- * Implements hook_entity_info_alter().
- *
- * @see apachesolr_search_entity_info_alter()
+ * Hook into the field operations
+ * - we want to save the same data in a shadow copy table for easier indexing.
+ * - We do not delete the file / media entity when its usage count goes to 0
+ * but instead we set status to 0.
+ * - This is meant to make the backport to drupal 6 easier
*/
-function apachesolr_attachments_entity_info_alter(&$entity_info) {
- $entity_info['file']['apachesolr']['result callback'] = 'apachesolr_attachments_file_result';
+function apachesolr_attachments_field_attach_insert($parent_entity_type, $parent_entity) {
+ apachesolr_attachments_field_attach_update($parent_entity_type, $parent_entity);
+}
+
+function apachesolr_attachments_field_attach_update($parent_entity_type, $parent_entity) {
+ foreach (field_info_instances($parent_entity_type, $parent_entity->type) as $instance) {
+ $field_info = field_info_field($instance['field_name']);
+ if ($field_info['type'] == 'file') {
+ // Include the file after the if, otherwise it'll get included everywhere
+ module_load_include('inc', 'apachesolr_attachments', 'apachesolr_attachments.index');
+
+ $items = field_get_items($parent_entity_type, $parent_entity, $field_info['field_name']);
+ foreach ($items as $file_info) {
+ $file = file_load($file_info['fid']);
+ // Discard empty entities
+ if (empty($file)) {
+ continue;
+ }
+ // Retrieve parent entity id and add its file usage
+ list($parent_entity_id) = entity_extract_ids($parent_entity_type, $parent_entity);
+ apachesolr_attachments_add_file_usage($file, $parent_entity_type, $parent_entity_id);
+ }
+ }
+ }
+}
+
+function apachesolr_attachments_field_attach_delete($parent_entity_type, $parent_entity) {
+ foreach (field_info_instances($parent_entity_type, $parent_entity->type) as $instance) {
+ $field_info = field_info_field($instance['field_name']);
+ if ($field_info['type'] == 'file') {
+ // Include the file after the if, otherwise it'll get included everywhere
+ module_load_include('inc', 'apachesolr_attachments', 'apachesolr_attachments.index');
+
+ $items = field_get_items($parent_entity_type, $parent_entity, $field_info['field_name']);
+ foreach ($items as $file_info) {
+ $file = file_load($file_info['fid']);
+ // Discard empty entities
+ if (empty($file)) {
+ continue;
+ }
+ // Retrieve parent entity id and delete its file usage
+ list($parent_entity_id) = entity_extract_ids($parent_entity_type, $parent_entity);
+ apachesolr_attachments_delete_file_usage($file, $parent_entity_type, $parent_entity_id);
+ }
+ }
+ }
}
/**
@@ -302,7 +449,7 @@ function theme_apachesolr_search_snippets__file($vars) {
$doc = $vars['doc'];
$snippets = $vars['snippets'];
- $node_link = t('
attached to: !node_link', array('!node_link' => l($doc->ss_file_node_title, 'node/' . $doc->is_nid)));
+ $node_link = t('
attached to: !node_link', array('!node_link' => l($doc->ss_file_entity_title, 'node/' . $doc->is_parent_entity_id)));
if (module_exists('file')) {
$file_type = t('!icon @filemime', array('@filemime' => $doc->ss_filemime, '!icon' => theme('file_icon', array('file' => (object) array('filemime' => $doc->ss_filemime)))));
}
@@ -323,3 +470,47 @@ function apachesolr_attachments_default_excluded() {
$default = array('aif', 'art', 'avi', 'bmp', 'gif', 'ico', 'jpg', 'mov', 'mp3', 'mp4', 'mpg', 'oga', 'ogv', 'png', 'psd', 'ra', 'ram', 'rgb', 'tif');
return $default;
}
+
+
+
+class ApachesolrAttachmentsEntityFieldQuery extends EntityFieldQuery {
+ /**
+ * Finishes the query.
+ *
+ * Adds tags, metaData, range and returns the requested list or count.
+ *
+ * @param SelectQuery $select_query
+ * A SelectQuery which has entity_type, entity_id, revision_id and bundle
+ * fields added.
+ * @param $id_key
+ * Which field's values to use as the returned array keys.
+ *
+ * @return
+ * See EntityFieldQuery::execute().
+ */
+ function finishQuery($select_query, $id_key = 'entity_id') {
+ foreach ($this->tags as $tag) {
+ $select_query->addTag($tag);
+ }
+ foreach ($this->metaData as $key => $object) {
+ $select_query->addMetaData($key, $object);
+ }
+ $select_query->addMetaData('entity_field_query', $this);
+ if ($this->range) {
+ $select_query->range($this->range['start'], $this->range['length']);
+ }
+ if ($this->count) {
+ return $select_query->countQuery()->execute()->fetchField();
+ }
+ $return = array();
+ foreach ($select_query->execute() as $partial_entity) {
+ $bundle = isset($partial_entity->bundle) ? $partial_entity->bundle : NULL;
+ $entity = entity_create_stub_entity($partial_entity->entity_type, array($partial_entity->entity_id, $partial_entity->revision_id, $bundle));
+ // This is adding the file id using our metaData field.
+ $entity->file_fid = $partial_entity->{$this->metaData['field_fid_key']};
+ $return[$partial_entity->entity_type][$partial_entity->$id_key] = $entity;
+ $this->ordered_results[] = $partial_entity;
+ }
+ return $return;
+ }
+}
\ No newline at end of file