diff --git a/apachesolr.admin.inc b/apachesolr.admin.inc index 0fa4d5c..d0e6833 100644 --- a/apachesolr.admin.inc +++ b/apachesolr.admin.inc @@ -259,6 +259,14 @@ function apachesolr_settings($form, &$form_state) { // Define all the Operations $confs = array(); $ops = array(); + // If the indexer is enabled add an index link + $confs['Index'] = array( + 'class' => 'operation', + 'data' => l(t('Index'), + 'admin/config/search/apachesolr/settings/' . $data['env_id'] . '/index', + array('query' => array('destination' => current_path())) + ), + ); // Whenever facetapi is enabled we also enable our operation link if (module_exists('facetapi')) { $confs['facets'] = array( @@ -294,7 +302,7 @@ function apachesolr_settings($form, &$form_state) { array('query' => array('destination' => $_GET['q'])) ), ); - $env_name = l(check_plain($data['name']), 'admin/config/search/apachesolr/settings/' . $data['env_id'] . '/index', array('query' => array('destination' => $_GET['q']))); + $env_name = l(check_plain($data['name']), 'admin/config/search/apachesolr/settings/' . $data['env_id'] . '/status', array('query' => array('destination' => $_GET['q']))); // Is this row our default environment? if ($environment_id == $default_environment) { $env_name = t('!environment (Default)', array('!environment' => $env_name)); @@ -409,8 +417,8 @@ function apachesolr_settings($form, &$form_state) { /** * Gets information about the fields already in solr index. */ -function apachesolr_index_page($environment = NULL) { - +function apachesolr_status_page($environment = NULL) { + module_load_include('inc', 'apachesolr', 'apachesolr.index'); if (empty($environment)) { $env_id = apachesolr_default_environment(); $environment = apachesolr_environment_load($env_id); @@ -427,10 +435,7 @@ function apachesolr_index_page($environment = NULL) { try { $solr = apachesolr_get_solr($environment["env_id"]); - // TODO: possibly clear this every page view if we are running multi-site. - if (apachesolr_index_get_last_updated()) { - $solr->clearCache(); - } + $solr->clearCache(); $data = $solr->getLuke(); } catch (Exception $e) { @@ -456,13 +461,12 @@ function apachesolr_index_page($environment = NULL) { drupal_set_message(t('Your schema.xml version is too old. You must update it and re-index your content.'), 'error'); } - $status = apachesolr_index_status('apachesolr_search'); + $status = apachesolr_index_status(); $indexed_message = t('@num Items (%percentage)', array( '%percentage' => ((int)min(100, 100 * ($status['total'] - $status['remaining']) / max(1, $status['total']))) . '%', '@num' => $data->index->numDocs, )); $messages[] = array('Indexed', $indexed_message); - $remaining_message = t('@items', array('@items' => format_plural($status['remaining'], t('1 item'), t('@count items')))); $messages[] = array('Remaining', $remaining_message); @@ -489,34 +493,19 @@ function apachesolr_index_page($environment = NULL) { $output['viewmore'] = array( '#markup' => l(t('View more details on the search index contents'), 'admin/reports/apachesolr'), ); - - $write_status = apachesolr_environment_variable_get($env_id, 'apachesolr_read_only', APACHESOLR_READ_WRITE); - $default_env_id = apachesolr_default_environment(); - if (($write_status == APACHESOLR_READ_WRITE) && ($env_id == $default_env_id)) { - // Display the Delete Index form. - $output['apachesolr_index_action_form'] = array( - '#prefix' => '

' . t('Search Index Actions') . '

', - 'form' => drupal_get_form('apachesolr_index_action_form'), - ); - } - else { - drupal_set_message(t('Options for deleting and re-indexing are not available. This could be because the index is read-only or you are not viewing the default environment. This can be changed on the settings page', array('!settings_page' => url('admin/config/search/apachesolr/settings/' . $env_id, array('query' => drupal_get_destination())))), 'warning'); - } + $output['index_action_form'] = drupal_get_form('apachesolr_index_action_form'); + $output['index_config_form'] = drupal_get_form('apachesolr_index_config_form'); return $output; } function apachesolr_index_report($env_id = NULL) { - if (empty($env_id)) { $env_id = apachesolr_default_environment(); } try { $solr = apachesolr_get_solr($env_id); - // TODO: possibly clear this every page view if we are running multi-site. - if (apachesolr_index_get_last_updated()) { - $solr->clearCache(); - } + $solr->clearCache(); $data = $solr->getLuke(); } catch (Exception $e) { @@ -583,140 +572,6 @@ function apachesolr_index_report($env_id = NULL) { } /** - * Create a form for deleting the contents of the Solr index. - */ -function apachesolr_index_action_form() { - $form = array(); - // Jump through some forms hoops to get a description under each radio button. - $actions = array( - 'remaining' => array( - 'title' => t('Index Queued content'), - 'description' => t('Could take time and could put an increased load on your server.'), - ), - 'reindex' => array( - 'title' => t('Queue all content for reindexing'), - ), - 'delete' => array( - 'title' => t('Delete the Search & Solr index'), - 'description' => t('Useful with a corrupt index or a new schema.xml.'), - ), - ); - foreach ($actions as $key => $action) { - // Generate the parents as the autogenerator does, so we will have a - // unique id for each radio button. - $form['action'][$key] = array( - '#type' => 'radio', - '#title' => check_plain($action['title']), - '#default_value' => 'remaining', - '#return_value' => $key, - '#parents' => array('action'), - '#description' => (!empty($action['description'])) ? check_plain($action['description']) : '', - '#id' => drupal_clean_css_identifier('edit-' . implode('-', array('action', $key))), - ); - } - - $form['submit'] = array( - '#type' => 'submit', - '#value' => t('Run'), - '#submit' => array('apachesolr_index_action_form_submit'), - ); - - return $form; -} - -/** - * Submit function for the index action form. - */ -function apachesolr_index_action_form_submit($form, &$form_state) { - switch ($form_state['values']['action']) { - case 'remaining': - apachesolr_batch_index_remaining(); - break; - - case 'reindex': - $form_state['redirect'] = 'admin/config/search/apachesolr/index/confirm/clear'; - break; - - case 'delete': - $form_state['redirect'] = 'admin/config/search/apachesolr/index/confirm/delete'; - break; - } -} - -/** - * Confirmation form for "Re-index all content" button - * @see apachesolr_clear_index_submit() - */ -function apachesolr_clear_index_confirm() { - $form = array(); - return confirm_form($form, t('Are you sure you want to queue content for reindexing?'), 'admin/config/search/apachesolr/index', t('All content on the site will be queued for indexing. The documents currently in the Solr index will remain searchable. The content will be gradually resubmitted to Solr during cron runs.'), t('Reindex'), t('Cancel')); -} - -/** - * Submit function for the "Re-index all content" confirmation form. - * - * @see apachesolr_clear_index_confirm() - */ -function apachesolr_clear_index_confirm_submit($form, &$form_state) { - $form_state['redirect'] = 'admin/config/search/apachesolr/index'; - apachesolr_rebuild_index_table(); - drupal_set_message(t('All the content on your site is queued for indexing. You can wait for it to be indexed during cron runs, or you can manually reindex it.'), 'warning'); -} - -/** - * Confirmation form for "Delete the index" button - * @see apachesolr_delete_index_submit() - */ -function apachesolr_delete_index_confirm() { - $form = array(); - return confirm_form($form, t('Are you sure you want to delete the search index?'), 'admin/config/search/apachesolr/index', t("All documents in the Solr index will be deleted. This is rarely necessary unless your index is corrupt or you have installed a new schema.xml. After doing this your content will need to be queued for indexing."), t('Delete'), t('Cancel')); -} - -/** - * Submit function for the "Delete the index" confirmation form. - * - * @see apachesolr_delete_index_confirm() - */ -function apachesolr_delete_index_confirm_submit($form, &$form_state) { - $form_state['redirect'] = 'admin/config/search/apachesolr/index'; - try { - apachesolr_delete_index(); - drupal_set_message(t('The Apache Solr content index has been erased. All the content on your site is queued for indexing. You can wait for it to be indexed during cron runs, or you can manually reindex it.'), 'warning'); - } - catch (Exception $e) { - watchdog('Apache Solr', nl2br(check_plain($e->getMessage())), NULL, WATCHDOG_ERROR); - } -} - -/** - * Utility function to delete the index and reset all index counters. - * - * @param $type - * a single content type to be deleted from the index. - * - * @throws Exception - */ -function apachesolr_delete_index($type = NULL) { - // Instantiate a new Solr object. - $solr = apachesolr_get_solr(); - if ($type) { - $query = 'bundle:' . $type; - } - else { - $query = '*:*'; - } - // Allow other modules to modify the delete query. - // For example, use the site hash so that you only delete this site's - // content: $query = 'hash:' . apachesolr_site_hash() - drupal_alter('apachesolr_delete_index', $query); - $solr->deleteByQuery($query); - $solr->commit(); - // Rebuild our node-tracking table. - apachesolr_rebuild_index_table(); - apachesolr_index_set_last_updated(REQUEST_TIME); -} - -/** * Page callback to show available conf files. */ function apachesolr_config_files_overview() { @@ -802,18 +657,213 @@ function apachesolr_config_file($name) { } /** - * Batch reindex functions. + * Form builder for the Apachesolr Indexer actions form. + * + * @see apachesolr_index_action_form_delete_submit(). */ +function apachesolr_index_action_form() { + $form = array(); + $form['action'] = array( + '#type' => 'fieldset', + '#title' => t('Index Modifications'), + '#collapsible' => TRUE, + ); + + $form['action']['remaining'] = array( + '#prefix' => '
', + '#suffix' => '
', + '#type' => 'submit', + '#value' => t('Index queued content'), + '#submit' => array('apachesolr_index_action_form_remaining_submit'), + ); + $form['action']['remaining_description'] = array( + '#prefix' => '
', + '#suffix' => '
', + '#markup' => t('Could take time and could put an increased load on your server.'), + ); + + $form['action']['reset'] = array( + '#prefix' => '
', + '#suffix' => '
', + '#type' => 'submit', + '#value' => t('Queue all content for reindexing'), + '#submit' => array('apachesolr_index_action_form_reset_submit'), + ); + $form['action']['reset_description'] = array( + '#prefix' => '
', + '#suffix' => '
', + '#markup' => ' ', + ); + $form['action']['delete'] = array( + '#prefix' => '
', + '#suffix' => '
', + '#type' => 'submit', + '#value' => t('Delete the Search & Solr index'), + '#submit' => array('apachesolr_index_action_form_delete_submit'), + ); + $form['action']['delete_description'] = array( + '#prefix' => '
', + '#suffix' => '
', + '#markup' => t('Useful with a corrupt index or a new schema.xml.'), + ); + + return $form; +} + +/** + * Submit handler for the Indexer actions form, delete button. + */ +function apachesolr_index_action_form_remaining_submit($form, &$form_state) { + $destination = array(); + if (isset($_GET['destination'])) { + $destination = drupal_get_destination(); + unset($_GET['destination']); + } + $env_id = apachesolr_default_environment(); + $form_state['redirect'] = array('admin/config/search/apachesolr/settings/' . $env_id . '/index/remaining', array('query' => $destination)); +} + +/** + * Submit handler for the Indexer actions form, delete button. + */ +function apachesolr_index_action_form_delete_submit($form, &$form_state) { + $destination = array(); + if (isset($_GET['destination'])) { + $destination = drupal_get_destination(); + unset($_GET['destination']); + } + + $env_id = apachesolr_default_environment(); + $form_state['redirect'] = array('admin/config/search/apachesolr/settings/' . $env_id . '/index/delete', array('query' => $destination)); +} + +/** + * Submit handler for the Indexer actions form, delete button. + */ +function apachesolr_index_action_form_reset_submit($form, &$form_state) { + $destination = array(); + if (isset($_GET['destination'])) { + $destination = drupal_get_destination(); + unset($_GET['destination']); + } + $env_id = apachesolr_default_environment(); + $form_state['redirect'] = array('admin/config/search/apachesolr/settings/' . $env_id . '/index/reset', array('query' => $destination)); +} + +/** + * Form builder for to reindex the remaining. + * + * @see apachesolr_index_action_form_delete_confirm_submit(). + */ +function apachesolr_index_action_form_remaining_confirm($form, &$form_state) { + $env_id = apachesolr_default_environment(); + return confirm_form($form, + t('Are you sure you want reindex your index?'), + 'admin/config/search/apachesolr/settings/' . $env_id . '/index', + NULL, + t('Remaining index') + ); +} + +/** + * Form builder for the index delete/clear form. + * + * @see apachesolr_index_action_form_delete_confirm_submit(). + */ +function apachesolr_index_action_form_delete_confirm($form, &$form_state) { + $env_id = apachesolr_default_environment(); + return confirm_form($form, + t('Are you sure you want to clear your index?'), + 'admin/config/search/apachesolr/settings/' . $env_id . '/index', + t('This will remove all data from your index and all search results will be incomplete until your site is reindexed.'), + t('Delete index') + ); +} + +/** + * Form builder for the index re-enqueue form. + * + * @see apachesolr_index_action_form_reset_confirm_submit(). + */ +function apachesolr_index_action_form_reset_confirm($form, &$form_state) { + $env_id = apachesolr_default_environment(); + return confirm_form($form, + t('Are you sure you want to queue content for reindexing?'), + 'admin/config/search/apachesolr/settings/' . $env_id . '/index', + t('All content on the site will be queued for indexing. The documents currently in the Solr index will remain searchable. The content will be gradually resubmitted to Solr during cron runs.'), + t('Reindex') + ); +} + +/** + * Submit handler for the deletion form. + */ +function apachesolr_index_action_form_remaining_confirm_submit($form, &$form_state) { + $form_state['redirect'] = 'admin/config/search/apachesolr'; + apachesolr_index_batch_index_remaining(); +} + +/** + * Submit handler for the deletion form. + */ +function apachesolr_index_action_form_reset_confirm_submit($form, &$form_state) { + $form_state['redirect'] = 'admin/config/search/apachesolr'; + apachesolr_index_mark_all_for_reindex(); + drupal_set_message(t('All the content on your site is queued for indexing. You can wait for it to be indexed during cron runs, or you can manually reindex it.')); +} + +function apachesolr_index_mark_all_for_reindex() { + module_load_include('inc', 'apachesolr', 'apachesolr.index'); + foreach (entity_get_info() as $entity_type => $entity_info) { + if ($entity_info['apachesolr']['indexable']) { + $bundles = apachesolr_index_get_bundles('default', $entity_type); + $reindex_callback = ''; + if (!empty($bundles)) { + $callback = apachesolr_entity_get_callback($entity_type, 'reindex callback'); + $reindex_callback = $callback; + } + if (! empty($reindex_callback)) { + if (! $reindex_callback()) { + drupal_set_message(t('There was an error reindexing @entity_type. Please consult the log for more information.', array('@entity_type' => $entity_info['label'])), 'error'); + return; + } + } + } + } +} + +/** + * Submit handler for the deletion form. + */ +function apachesolr_index_action_form_delete_confirm_submit($form, &$form_state) { + // Instantiate a new Solr object. + $solr = apachesolr_get_solr(); + $query = '*:*'; + + // Allow other modules to modify the delete query. + // For example, use the site hash so that you only delete this site's + // content: $query = 'hash:' . apachesolr_site_hash() + drupal_alter('apachesolr_delete_index', $query); + $solr->deleteByQuery($query); + $solr->commit(); + + // Rebuild our tracking table. + variable_get('apachesolr_index_last_run', 0); + + drupal_set_message(t('The index has been deleted.')); + + $form_state['redirect'] = 'admin/config/search/apachesolr'; +} /** * Submit a batch job to index the remaining, unindexed content. */ -function apachesolr_batch_index_remaining() { +function apachesolr_index_batch_index_remaining() { $batch = array( 'operations' => array( - array('apachesolr_batch_index_nodes', array()), + array('apachesolr_index_batch_index_entities', array()), ), - 'finished' => 'apachesolr_batch_index_finished', + 'finished' => 'apachesolr_index_batch_index_finished', 'title' => t('Indexing'), 'init_message' => t('Preparing to submit content to Solr for indexing...'), 'progress_message' => t('Submitting content to Solr...'), @@ -826,7 +876,8 @@ function apachesolr_batch_index_remaining() { /** * Batch Operation Callback */ -function apachesolr_batch_index_nodes(&$context) { +function apachesolr_index_batch_index_entities(&$context) { + module_load_include('inc', 'apachesolr', 'apachesolr.index'); if (empty($context['sandbox'])) { try { // Get the $solr object @@ -841,7 +892,7 @@ function apachesolr_batch_index_nodes(&$context) { return FALSE; } - $status = apachesolr_index_status('apachesolr_search'); + $status = apachesolr_index_status(); $context['sandbox']['progress'] = 0; $context['sandbox']['max'] = $status['remaining']; } @@ -851,10 +902,17 @@ function apachesolr_batch_index_nodes(&$context) { $limit = variable_get('apachesolr_cron_limit', 50); // With each pass through the callback, retrieve the next group of nids. - $rows = apachesolr_get_nodes_to_index('apachesolr_search', $limit); - $pos = apachesolr_index_nodes($rows, 'apachesolr_search'); + $rows = apachesolr_index_get_entities_to_index($limit); + $documents = array(); + $entities_processed = 0; + foreach ($rows as $row) { + $documents = array_merge($documents, apachesolr_index_index_entity($row)); + $entities_processed++; + } + apachesolr_index_send_to_solr($documents); + apachesolr_index_set_last_updated(REQUEST_TIME); - $context['sandbox']['progress'] += $pos['nodes_processed']; + $context['sandbox']['progress'] += $entities_processed; $context['message'] = t('Indexed @current of @total nodes', array('@current' => $context['sandbox']['progress'], '@total' => $context['sandbox']['max'])); // Inform the batch engine that we are not finished, and provide an @@ -871,7 +929,7 @@ function apachesolr_batch_index_nodes(&$context) { /** * Batch 'finished' callback */ -function apachesolr_batch_index_finished($success, $results, $operations) { +function apachesolr_index_batch_index_finished($success, $results, $operations) { $message = ''; // $results['count'] will not be set if Solr is unavailable. if (isset($results['count'])) { @@ -886,5 +944,79 @@ function apachesolr_batch_index_finished($success, $results, $operations) { $message .= ' ' . t('An error occurred while processing @num with arguments :', array('@num' => $error_operation[0])) . print_r($error_operation[0], TRUE); $type = 'error'; } - drupal_set_message(check_plain($message), $type); + drupal_set_message($message, $type); +} + + +/** + * Form builder for the bundle configuration form. + * + * @see apachesolr_index_config_form_submit(). + */ +function apachesolr_index_config_form() { + $form['config'] = array( + '#type' => 'fieldset', + '#title' => t('Index Configurations'), + '#collapsible' => TRUE, + ); + + $form['config']['bundles'] = array( + '#type' => 'markup', + '#markup' => t('Select the entity types and bundles that should be indexed.'), + ); + + // For future extensibility, when we have multiple cores. + $form['config']['core'] = array( + '#type' => 'value', + '#value' => 'default', + ); + + foreach (entity_get_info() as $entity_type => $entity_info) { + if (!empty($entity_info['apachesolr']['indexable'])) { + $options = array(); + foreach ($entity_info['bundles'] as $key => $info) { + $options[$key] = $info['label']; + } + + $form['config']['entities']['#tree'] = TRUE; + + $form['config']['entities'][$entity_type] = array( + '#type' => 'checkboxes', + '#title' => check_plain($entity_info['label']), + '#options' => $options, + '#default_value' => apachesolr_index_get_bundles('default', $entity_type), + ); + } + } + + $form['config']['submit'] = array('#type' => 'submit', '#value' => t('Save')); + + return $form; +} + +/** + * Submit handler for the bundle configuration form. + */ +function apachesolr_index_config_form_submit($form, &$form_state) { + $form_values = $form_state['values']; + $core = $form_values['core']; + $entity_count = 0; + + foreach ($form_values['entities'] as $entity_type => $bundles) { + $entity_bundles = array_flip($bundles); + unset($entity_bundles[0]); // Nuke unchecked bundles + if (!empty($entity_bundles)) { + $entity_count++; + } + apachesolr_index_set_bundles($core, $entity_type, $entity_bundles); + } + + // set our amount of entities we are indexing. Useful for cron limits + // to make sure every entity can be indexed + variable_set('apachesolr_index_entity_count', $entity_count); + + // Clear the entity cache, since we will be changing its data. + entity_info_cache_clear(); + + drupal_set_message(t('Your settings have been saved. You should reindex your site for the changes to take effect.')); } diff --git a/apachesolr.index.inc b/apachesolr.index.inc index a51b98d..f81e55b 100644 --- a/apachesolr.index.inc +++ b/apachesolr.index.inc @@ -6,148 +6,721 @@ */ /** - * Given a node, return a document representing that node. + * Determines if we should index the provided entity. + * + * Whether or not a given entity is indexed is determined on a per-bundle basis. + * Entities/Bundles that have no index flag are presumed to not get indexed. + * + * @param stdClass $entity + * The entity we may or may not want to index. + * @param string $type + * The type of entity. + * @return boolean + * TRUE if this entity should be indexed, FALSE otherwise. */ -function apachesolr_node_to_document($node, $namespace) { - $document = FALSE; - // Let any module exclude this node from the index. - $build_document = TRUE; - foreach (module_implements('apachesolr_node_exclude') as $module) { - $exclude = module_invoke($module, 'apachesolr_node_exclude', $node, $namespace); - if (!empty($exclude)) { - $build_document = FALSE; - } - } - - if ($build_document) { - $document = new ApacheSolrDocument(); - $document->id = apachesolr_document_id($node->nid); - $document->site = url(NULL, array('absolute' => TRUE)); - $document->hash = apachesolr_site_hash(); - $document->entity_type = 'node'; - $document->entity_id = $node->nid; - $document->bundle = $node->type; - $document->bundle_name = node_type_get_name($node); - $document->label = apachesolr_clean_text($node->title); - $path = 'node/' . $node->nid; - $document->url = url($path, array('absolute' => TRUE)); - $document->path = $path; - // Build the node body. - $build = node_view($node, 'search_index'); - // Why do we need this? - unset($build['#theme']); - $text = drupal_render($build); - $document->content = apachesolr_clean_text($text); - if (isset($node->teaser)) { - $document->teaser = apachesolr_clean_text($node->teaser); +function apachesolr_entity_should_index($entity, $type) { + $info = entity_get_info($type); + list($id, $vid, $bundle) = entity_extract_ids($type, $entity); + + if ($bundle && isset($info['bundles'][$bundle]['apachesolr']['index']) && $info['bundles'][$bundle]['apachesolr']['index']) { + return TRUE; + } + + return FALSE; +} + +/** + * Getting the default callbacks for the entity index functions + */ +function apachesolr_index_get_entity_defaults() { + // Set those values that we know. Other modules can do so + // for their own entities if they want. + $entity_info = array(); + $entity_info['node']['indexable'] = TRUE; + $entity_info['node']['status callback'] = 'apachesolr_index_node_status_callback'; + $entity_info['node']['document callback'][] = 'apachesolr_index_node_solr_document'; + $entity_info['node']['reindex callback'] = 'apachesolr_index_node_solr_reindex'; + $entity_info['node']['index_table'] = 'apachesolr_index_entities_node'; + + //Allow implementations of HOOK_apachesolr_index_get_entity_defaults_alter to modify these default indexers + drupal_alter('apachesolr_index_get_entity_defaults', $entity_info); + return $entity_info; +} + +/** + * Helper function for modules implementing hook_search's 'status' op. + */ +function apachesolr_index_status() { + $remaining = 0; + $total = 0; + $last_change = variable_get('apachesolr_index_last_run', 0); + foreach (entity_get_info() as $type => $info) { + $bundles = array(); + foreach ($info['bundles'] as $bundle => $bundle_info) { + if ($bundle && isset($info['bundles'][$bundle]['apachesolr']['index']) && $info['bundles'][$bundle]['apachesolr']['index']) { + $bundles[] = $bundle; + } + } + // If we're not checking any bundles of this entity type, just skip them all. + if (empty($bundles)) { + continue; + } + + if (isset($info['apachesolr']['index_table'])) { + $table = $info['apachesolr']['index_table']; } else { - $document->teaser = truncate_utf8($document->content, 300, TRUE); - } - // Path aliases can have important information about the content. - // Add them to the index as well. - if (function_exists('drupal_get_path_alias')) { - // Add any path alias to the index, looking first for language specific - // aliases but using language neutral aliases otherwise. - $language = empty($node->language) ? NULL : $node->language; - $output = drupal_get_path_alias($path, $language); - if ($output && $output != $path) { - $document->path_alias = $output; + $table = 'apachesolr_index_entities'; + } + $query = db_select($table, 'asn')->condition('asn.status', 1); + $total += $query->countQuery()->execute()->fetchField(); + + $last_entity_id = variable_get('apachesolr_index_last_run_id_' . $type, 0); + + $query = db_select($table, 'asn') + ->condition('asn.status', 1) + ->condition(db_or() + ->condition('asn.changed', $last_change, '>') + ->condition(db_and() + ->condition('asn.changed', $last_change) + ->condition('asn.entity_id', $last_entity_id, '>'))); + $remaining += $query->countQuery()->execute()->fetchField(); + } + + return array('remaining' => $remaining, 'total' => $total); +} + +/** + * Worker callback for apachesolr_index_entities. + * + * @see apachesolr_index_nodes() for the old-skool version. + * @return array of documents that need to be sent to the index of solr + */ +function apachesolr_index_index_entity($item) { + + // Always build the content for the index as an anonynmous user to avoid + // exposing restricted fields and such. + // @todo Uncomment these lines when we're done debugging, since they break dpm(). + global $user; + drupal_save_session(FALSE); + $saved_user = $user; + $user = drupal_anonymous_user(); + + // Pull out all of our pertinent data. + $entity_type = $item->entity_type; + + // TRUE on reset to bypass static caching and not blow out our memory limit. + $entity = entity_load($entity_type, array($item->entity_id), array(), TRUE); + $entity = $entity ? reset($entity) : FALSE; + + if (empty($entity)) { + // If the object failed to load, just stop. + return FALSE; + } + + list($entity_id, $vid, $bundle) = entity_extract_ids($entity_type, $entity); + + //$status = apachesolr_entity_get_callback($entity, $type, 'status callback', array($entity, $type)); + + // Create a new document, and do the bare minimum on it. + $document = _apachesolr_index_process_entity_get_document($entity, $entity_type); + + //Get the callback array to add stuff to the document + $callbacks = apachesolr_entity_get_callback($entity_type, 'document callback'); + $documents = array(); + foreach ($callbacks as $callback) { + // Call a type-specific callback to add stuff to the document. + $documents = array_merge($documents, $callback($document, $entity, $entity_type)); + } + + //do this for all possible documents that were returned by the callbacks + foreach ($documents as $id => $document) { + // Call an all-entity hook to add stuff to the document. + module_invoke_all('apachesolr_index_document_build', $documents[$id], $entity, $entity_type); + + // Call a type-specific hook to add stuff to the document. + module_invoke_all('apachesolr_index_document_' . $entity_type . '_build', $documents[$id], $entity, $entity_type); + + // Call a bundle-specific hook if available. + module_invoke_all('apachesolr_index_document_' . $entity_type . '_' . $bundle . '_build', $documents[$id], $entity, $entity_type); + + // Now allow modules to alter each other's additions for maximum flexibility. + drupal_alter('apachesolr_index_document', $documents[$id], $entity, $entity_type); + drupal_alter('apachesolr_index_document_' . $entity_type, $documents[$id], $entity, $entity_type); + drupal_alter('apachesolr_index_document_' . $entity_type . '_' . $bundle, $documents[$id], $entity, $entity_type); + // Final processing to ensure that the document is properly structured. + + // All records must have a label field, which is used for user-friendly labeling. + if (empty($documents[$id]->label)) { + $documents[$id]->label = ''; + } + + // All records must have a "body" field, which is used for fulltext indexing. + // If we don't have one, enter an empty value. This does mean that the entity + // will not be fulltext searchable. + if (empty($documents[$id]->content)) { + $documents[$id]->content = ''; + } + + // All records must have a "teaser" field, which is used for abbreviated + // displays when no highlighted text is available. + if (empty($documents[$id]->teaser)) { + $documents[$id]->teaser = truncate_utf8($documents[$id]->content, 300, TRUE); + } + + // Add additional indexing based on the body of each record. + apachesolr_index_add_tags_to_document($documents[$id], $documents[$id]->content); + } + + // Restore the user. + // @todo Uncomment these lines when we're done debugging, since they break dpm(). + $user = $saved_user; + drupal_save_session(TRUE); + + return $documents; + +} + +/** + * Set the timestamp of the last index update + * @param $updated + * A timestamp or zero. If zero, the variable is deleted. + */ +function apachesolr_index_set_last_updated($timestamp = 0) { + variable_set('apachesolr_index_last_run', $timestamp); +} + +/** + * Get the timestamp of the last index update. + * @return integer (timestamp) + */ +function apachesolr_index_get_last_updated() { + return variable_get('apachesolr_index_last_run', 0); +} + +function apachesolr_index_send_to_solr($documents) { + try { + // Get the $solr object + $solr = apachesolr_get_solr(); + // If there is no server available, don't continue. + if (!$solr->ping(variable_get('apachesolr_ping_timeout', 4))) { + throw new Exception(t('No Solr instance available during indexing.')); + } + } + catch (Exception $e) { + watchdog('Apache Solr', nl2br(check_plain($e->getMessage())), NULL, WATCHDOG_ERROR); + return FALSE; + } + + // Send the document off to Solr. + watchdog('Apache Solr', 'Adding @count documents.', array('@count' => count($documents))); + try { + $docs_chunk = array_chunk($documents, 20); + foreach ($docs_chunk as $docs) { + $solr->addDocuments($docs); + } + $ret = $solr->addDocuments($documents); + watchdog('Apache Solr', 'Indexing succeeded on @count documents', array( + '@count' => count($documents), + ), WATCHDOG_INFO); + } + catch (Exception $e) { + if (!empty($docs)) { + foreach ($docs as $doc) { + $eids[] = $doc->entity_id; + } + } + watchdog('Apache Solr', 'Indexing failed on one of the following entity ids: @eids
!message', array( + '@eids' => implode(', ', $eids), + '!message' => nl2br(strip_tags($e->getMessage())), + ), WATCHDOG_ERROR); + return FALSE; + } +} + +/** + * Extract HTML tag contents from $text and add to boost fields. + * + * $text must be stripped of control characters before hand. + */ +function apachesolr_index_add_tags_to_document(&$document, $text) { + $tags_to_index = variable_get('apachesolr_tags_to_index', array( + 'h1' => 'tags_h1', + 'h2' => 'tags_h2_h3', + 'h3' => 'tags_h2_h3', + 'h4' => 'tags_h4_h5_h6', + 'h5' => 'tags_h4_h5_h6', + 'h6' => 'tags_h4_h5_h6', + 'u' => 'tags_inline', + 'b' => 'tags_inline', + 'i' => 'tags_inline', + 'strong' => 'tags_inline', + 'em' => 'tags_inline', + 'a' => 'tags_a' + )); + + // Strip off all ignored tags. + $text = strip_tags($text, '<' . implode('><', array_keys($tags_to_index)) . '>'); + + preg_match_all('@<(' . implode('|', array_keys($tags_to_index)) . ')[^>]*>(.*)@Ui', $text, $matches); + foreach ($matches[1] as $key => $tag) { + $tag = strtolower($tag); + // We don't want to index links auto-generated by the url filter. + if ($tag != 'a' || !preg_match('@(?:http://|https://|ftp://|mailto:|smb://|afp://|file://|gopher://|news://|ssl://|sslv2://|sslv3://|tls://|tcp://|udp://|www\.)[a-zA-Z0-9]+@', $matches[2][$key])) { + if (!isset($document->{$tags_to_index[$tag]})) { + $document->{$tags_to_index[$tag]} = ''; } + $document->{$tags_to_index[$tag]} .= ' ' . apachesolr_clean_text($matches[2][$key]); } + } +} + + +/** + * Returns a generic Solr document object for this entity. + * + * This function will do the basic processing for the document that is common + * to all entities, but virtually all entities will need their own additional + * processing. + * + * @param stdClass $entity + * The entity for which we want a document. + * @param string $entity_type + * The type of entity we're processing. + * @return ApacheSolrDocument + */ +function _apachesolr_index_process_entity_get_document($entity, $entity_type) { + list($entity_id, $vid, $bundle) = entity_extract_ids($entity_type, $entity); + + $document = new ApacheSolrDocument(); + + $document->id = apachesolr_document_id($entity_id, $entity_type); + $document->site = url(NULL, array('absolute' => TRUE)); + $document->hash = apachesolr_site_hash(); - $document->ss_name = $node->name; - // We want the name to ale be searchable for keywords. - $document->tos_name = $node->name; + $document->entity_id = $entity_id; + $document->entity_type = $entity_type; + $document->bundle = $bundle; + $document->bundle_name = entity_bundle_label($entity_type, $bundle); + + $path = entity_uri($entity_type, $entity); + // A path is not a requirement of an entity + if (!empty($path)) { + $document->path = $path['path']; + $document->url = url($path['path'], $path['options'] + array('absolute' => TRUE)); + } + if (empty($entity->language)) { + // 'und' is the language-neutral code in Drupal 7. + $document->language = LANGUAGE_NONE; + } + else { + $document->language = $entity->language; + } - // Everything else uses dynamic fields - $document->is_uid = $node->uid; - $document->bs_status = $node->status; - $document->bs_sticky = $node->sticky; - $document->bs_promote = $node->promote; - $document->is_tnid = $node->tnid; - $document->bs_translate = $node->translate; - if (empty($node->language)) { - // 'und' is the language-neutral code in Drupal 7. - $document->ss_language = LANGUAGE_NONE; + // Path aliases can have important information about the content. + // Add them to the index as well. + if (function_exists('drupal_get_path_alias')) { + // Add any path alias to the index, looking first for language specific + // aliases but using language neutral aliases otherwise. + $output = drupal_get_path_alias($document->path, $document->language); + if ($output && $output != $document->path) { + $document->path_alias = $output; + } + } + return $document; +} + + + + +/** + * Returns an array of rows from a query based on an indexing namespace. + * @see apachesolr_get_nodes_to_index for old-skool version + */ +function apachesolr_index_get_entities_to_index($limit) { + $rows = array(); + if (variable_get('apachesolr_read_only', 0)) { + return $rows; + } + + $last_change = apachesolr_index_get_last_updated(); + foreach (entity_get_info() as $type => $info) { + $bundles = array(); + foreach ($info['bundles'] as $bundle => $bundle_info) { + if ($bundle && isset($info['bundles'][$bundle]['apachesolr']['index']) && $info['bundles'][$bundle]['apachesolr']['index']) { + $bundles[] = $bundle; + } + } + // If we're not checking any bundles of this entity type, just skip them all. + if (empty($bundles)) { + continue; + } + + if (isset($info['apachesolr']['index_table'])) { + $table = $info['apachesolr']['index_table']; } else { - $document->ss_language = $node->language; + $table = 'apachesolr_index_entities'; } - $document->ds_created = apachesolr_date_iso($node->created); - $document->ds_changed = apachesolr_date_iso($node->changed); - if (isset($node->last_comment_timestamp) && !empty($node->comment_count)) { - $document->ds_last_comment_timestamp = apachesolr_date_iso($node->last_comment_timestamp); - $document->ds_last_comment_or_change = apachesolr_date_iso(max($node->last_comment_timestamp, $node->changed)); + + $last_entity_id = variable_get('apachesolr_index_last_run_id_' . $type, 0); + + // Find the next batch of entities to index for this entity type. Note that + // for ordering we're grabbing the oldest first and then ordering by ID so + // that we get a definitive order. + + $query = db_select($table, 'aie') + ->fields('aie', array('entity_type', 'entity_id', 'changed')) + ->condition('aie.status', 1) + ->condition('aie.entity_type', $type) + ->condition('aie.bundle', $bundles) + ->condition(db_or()->condition('aie.changed', $last_change, '>') + ->condition('aie.changed', $last_change) + ->condition(db_and()->condition('aie.entity_id', $last_entity_id, '>'))) + ->orderBy('aie.changed') + ->orderBy('aie.entity_id') + ->range(0, $limit); + + $records = $query->execute()->fetchAll(); + + // @todo This assumes an int ID, which technically the entity system + // doesn't require. Fix this later. + $max_id = 0; + foreach ($records as $record) { + $rows[] = $record; + $max_id = max($max_id, $record->entity_id); + } + + //do not set it to 0 because this would enable reindexing of all the types + if (!empty($max_id)) { + variable_set('apachesolr_index_last_run_id_' . $type, $max_id); + } + } + return $rows; +} + +/** + * Delete an entity from the indexer. + */ +function apachesolr_index_delete_entity_from_index($entity_type, $entity) { + static $failed = FALSE; + if ($failed) { + return FALSE; + } + try { + list($id) = entity_extract_ids($entity_type, $entity); + $solr = apachesolr_get_solr(); + $solr->deleteById(apachesolr_document_id($id, $entity_type)); + apachesolr_index_set_last_updated(REQUEST_TIME); + return TRUE; + } + catch (Exception $e) { + watchdog('Apache Solr', nl2br(check_plain($e->getMessage())), NULL, WATCHDOG_ERROR); + // Don't keep trying queries if they are failing. + $failed = TRUE; + return FALSE; + } +} + +/** + * Retrieve the indexer table for an entity type. + */ +function apachesolr_index_get_indexer_table($type) { + $defaults = apachesolr_index_get_entity_defaults(); + if (isset($defaults[$type]['index_table'])) { + $indexer_table = $defaults[$type]['index_table']; } else { - $document->ds_last_comment_or_change = apachesolr_date_iso($node->changed); - } - $document->is_comment_count = isset($node->comment_count) ? $node->comment_count : 0; - - // Handle fields including taxonomy. - $indexed_fields = apachesolr_entity_fields('node'); - foreach ($indexed_fields as $index_key => $field_info) { - $field_name = $field_info['field']['field_name']; - // See if the node has fields that can be indexed - if (isset($node->{$field_name})) { - // Got a field. - $function = $field_info['indexing_callback']; - if ($function && is_callable($function)) { - // NOTE: This function should always return an array. One - // node field may be indexed to multiple Solr fields. - $fields = $function($node, $field_name, $index_key, $field_info); - foreach ($fields as $field) { - // It's fine to use this method also for single value fields. - $document->setMultiValue($field['key'], $field['value']); - } - } + $indexer_table = 'apachesolr_index_entities'; + } + return $indexer_table; +} + +/** + * Sets what bundles on the specified entity type should be indexed. + * + * @param string $core + * The Solr core for which to index entities. + * @param string $entity_type + * The entity type to index. + * @param array $bundles + * The machine names of the bundles to index. + */ +function apachesolr_index_set_bundles($core, $entity_type, array $bundles) { + $transaction = db_transaction(); + try { + db_delete('apachesolr_index_bundles') + ->condition('core', $core) + ->condition('entity_type', $entity_type) + ->execute(); + + if ($bundles) { + $insert = db_insert('apachesolr_index_bundles') + ->fields(array('core', 'entity_type', 'bundle')); + + foreach ($bundles as $bundle) { + $insert->values(array( + 'core' => $core, + 'entity_type' => $entity_type, + 'bundle' => $bundle, + )); } + $insert->execute(); } - // Index book module data. - if (!empty($node->book['bid'])) { - // Hard-coded - must change if apachesolr_index_key() changes. - $document->is_book_bid = (int) $node->book['bid']; - } - apachesolr_add_tags_to_document($document, $text); - - // Fetch extra data normally not visible, including comments. - // We do this manually (with module_implements instead of node_invoke_nodeapi) - // because we want a keyed array to come back. Only in this way can we decide - // whether to index comments or not. - $extra = array(); - $excludes = variable_get('apachesolr_exclude_nodeapi_types', array()); - $exclude_nodeapi = isset($excludes[$node->type]) ? $excludes[$node->type] : array(); - foreach (module_implements('node_update_index') as $module) { - // Invoke nodeapi if this module has not been excluded, for example, - // exclude 'comment' for a type to skip indexing its comments. - if (empty($exclude_nodeapi[$module])) { - $function = $module . '_node_update_index'; - if ($output = $function($node)) { - $extra[$module] = $output; - } + } + catch (Exception $e) { + $transaction->rollback(); + watchdog_exception('Apache Solr', $e); + } +} + +/** + * Gets a list of the bundles on the specified entity type that should be indexed. + * + * @param string $core + * The Solr core for which to index entities. + * @param string $entity_type + * The entity type to index. + * @return array + * The bundles that should be indexed. + */ +function apachesolr_index_get_bundles($core, $entity_type) { + // This could be called before the schema is installed so it is wise to + // Verify this + if (db_table_exists('apachesolr_index_bundles')) { + return db_query("SELECT bundle FROM {apachesolr_index_bundles} WHERE core = :core AND entity_type = :entity_type", array( + ':core' => $core, + ':entity_type' => $entity_type, + ))->fetchCol(); + } + return array(); +} + +// This really should be in core, but it isn't yet. When it gets added to core, +// we can remove this version. +// @see http://drupal.org/node/969180 +if (!function_exists('entity_bundle_label')) { + +/** + * Returns the label of a bundle. + * + * @param $entity_type + * The entity type; e.g. 'node' or 'user'. + * @param $entity + * The entity for which we want the human-readable label of its bundle. + * + * @return + * A string with the human-readable name of the bundle, or FALSE if not specified. + */ +function entity_bundle_label($entity_type, $bundle) { + $labels = &drupal_static(__FUNCTION__, array()); + + if (empty($labels)) { + foreach (entity_get_info() as $entity_type => $entity_info) { + foreach ($entity_info['bundles'] as $bundle => $bundle_info) { + $labels[$entity_type][$bundle] = !empty($bundle_info['label']) ? $bundle_info['label'] : FALSE; } } - if (isset($extra['comment'])) { - $comments = $extra['comment']; - unset($extra['comment']); - $document->ts_comments = apachesolr_clean_text($comments); - // @todo: do we want to reproduce apachesolr_add_tags_to_document() for comments? + } + + return $labels[$entity_type][$bundle]; +} + +} + + +/************************ + * The NODE entity indexing part + ************************/ + +/** + * Builds the node-specific information for a Solr document. + * + * @param ApacheSolrDocument $document + * The Solr document we are building up. + * @param stdClass $entity + * The entity we are indexing. + * @param string $entity_type + * The type of entity we're dealing with. + */ +function apachesolr_index_node_solr_document(ApacheSolrDocument $document, $node, $entity_type) { + + // None of these get added unless they are explicitly in our schema.xml + $document->label = apachesolr_clean_text($node->title); + + // Build the node body. + $build = node_view($node, 'search_index'); + // Why do we need this? + unset($build['#theme']); + $text = drupal_render($build); + $document->content = apachesolr_clean_text($text); + if (isset($node->teaser)) { + $document->teaser = apachesolr_clean_text($node->teaser); + } + else { + $document->teaser = truncate_utf8($document->content, 300, TRUE); + } + // Path aliases can have important information about the content. + // Add them to the index as well. + if (function_exists('drupal_get_path_alias')) { + // Add any path alias to the index, looking first for language specific + // aliases but using language neutral aliases otherwise. + $language = empty($node->language) ? NULL : $node->language; + $output = drupal_get_path_alias($path, $language); + if ($output && $output != $path) { + $document->path_alias = $output; + } + } + + $document->ss_name = $node->name; + // We want the name to ale be searchable for keywords. + $document->tos_name = $node->name; + + // Everything else uses dynamic fields + $document->is_uid = $node->uid; + $document->bs_status = $node->status; + $document->bs_sticky = $node->sticky; + $document->bs_promote = $node->promote; + $document->is_tnid = $node->tnid; + $document->bs_translate = $node->translate; + if (empty($node->language)) { + // 'und' is the language-neutral code in Drupal 7. + $document->ss_language = LANGUAGE_NONE; + } + else { + $document->ss_language = $node->language; + } + $document->ds_created = apachesolr_date_iso($node->created); + $document->ds_changed = apachesolr_date_iso($node->changed); + if (isset($node->last_comment_timestamp) && !empty($node->comment_count)) { + $document->ds_last_comment_timestamp = apachesolr_date_iso($node->last_comment_timestamp); + $document->ds_last_comment_or_change = apachesolr_date_iso(max($node->last_comment_timestamp, $node->changed)); + } + else { + $document->ds_last_comment_or_change = apachesolr_date_iso($node->changed); + } + $document->is_comment_count = isset($node->comment_count) ? $node->comment_count : 0; + + // Handle fields including taxonomy. + $indexed_fields = apachesolr_entity_fields('node'); + foreach ($indexed_fields as $index_key => $field_info) { + $field_name = $field_info['field']['field_name']; + // See if the node has fields that can be indexed + if (isset($node->{$field_name})) { + // Got a field. + $function = $field_info['indexing_callback']; + if ($function && is_callable($function)) { + // NOTE: This function should always return an array. One + // node field may be indexed to multiple Solr fields. + $fields = $function($node, $field_name, $index_key, $field_info); + foreach ($fields as $field) { + // It's fine to use this method also for single value fields. + $document->setMultiValue($field['key'], $field['value']); + } + } } - // Use an omit-norms text field since this is generally going to be short; not - // really a full-text field. - $document->tos_content_extra = apachesolr_clean_text(implode(' ', $extra)); + } + + // Fetch extra data normally not visible, including comments. + // We do this manually (with module_implements instead of node_invoke_nodeapi) + // because we want a keyed array to come back. Only in this way can we decide + // whether to index comments or not. + $extra = array(); + $exclude_comments = in_array($node->type, variable_get('apachesolr_exclude_comments_types', array()), TRUE); - // Let modules add to the document. - foreach (module_implements('apachesolr_update_index') as $module) { - $function = $module . '_apachesolr_update_index'; - $function($document, $node, $namespace); + // Invoke the hook_node_update_index hook + foreach (module_implements('node_update_index') as $module) { + if ($exclude_comments && $module == 'comment') { + // Don't add comments. + continue; + } + $function = $module . '_node_update_index'; + if ($output = $function($node)) { + $extra[$module] = $output; } } - return $document; + if (!variable_get('apachesolr_index_comments_with_node', TRUE)) { + unset($extra['comment']); + } + else { + $document->ts_comments = apachesolr_clean_text($comments); + // @todo: do we want to reproduce apachesolr_add_tags_to_document() for comments? + } + // Use an omit-norms text field since this is generally going to be short; not + // really a full-text field. + $document->tos_content_extra = apachesolr_clean_text(implode(' ', $extra)); + + + $document->type_name = node_type_get_name($node); + $document->created = apachesolr_date_iso($node->created); + $document->changed = apachesolr_date_iso($node->changed); + $last_change = (isset($node->last_comment_timestamp) && $node->last_comment_timestamp > $node->changed) ? $node->last_comment_timestamp : $node->changed; + $document->last_comment_or_change = apachesolr_date_iso($last_change); + $document->comment_count = isset($node->comment_count) ? $node->comment_count : 0; + + // We need to get the real username here, since it needs a full user object. + // That means we can't do the format_username() call on the display side. + $document->name = format_username(user_load($node->uid)); + + // Let modules add to the document. + foreach (module_implements('apachesolr_update_index') as $module) { + $function = $module . '_apachesolr_update_index'; + // @todo specify namespace + $namespace = 'apachesolr_search'; + $function($document, $node, $namespace); + } + + // Generic usecase for future reference. Callbacks can + // allow you to send back multiple documents + $documents = array(); + $documents[] = $document; + return $documents; +} + +/** + * Reindexing callback for ApacheSolr, for nodes. + */ +function apachesolr_index_node_solr_reindex() { + + $indexer_table = apachesolr_index_get_indexer_table('node'); + $transaction = db_transaction(); + try { + db_delete($indexer_table) + ->condition('entity_type', 'node') + ->execute(); + + $select = db_select('node', 'n'); + $select->addExpression("'node'", 'entity_type'); + $select->addField('n', 'nid', 'entity_id'); + $select->addField('n', 'type', 'bundle'); + $select->addField('n', 'status', 'status'); + $select->addExpression(REQUEST_TIME, 'changed'); + $select->condition('n.type', apachesolr_index_get_bundles('default', 'node'), 'IN'); + + $insert = db_insert($indexer_table) + ->fields(array('entity_id', 'bundle', 'status', 'entity_type', 'changed')) + ->from($select) + ->execute(); + } + catch (Exception $e) { + $transaction->rollback(); + //drupal_set_message($e->getMessage(), 'error'); + watchdog_exception('Apache Solr', $e); + return FALSE; + } + + return TRUE; +} + +/** + * Status callback for ApacheSolr, for nodes. + */ +function apachesolr_index_node_status_callback($node, $type) { + return $node->status; } + /** * Callback that converts term_reference field into an array */ diff --git a/apachesolr.install b/apachesolr.install index 0917d9f..8f772d4 100644 --- a/apachesolr.install +++ b/apachesolr.install @@ -65,7 +65,11 @@ function apachesolr_install() { apachesolr_search_mlt_save_block(array('name' => st('More like this'))); db_insert('apachesolr_environment')->fields(array('env_id' => 'solr', 'name' => 'localhost server', 'url' => 'http://localhost:8983/solr'))->execute(); drupal_set_message(st('Apache Solr is enabled. Visit the settings page.', array('@settings_link' => url('admin/config/search/apachesolr')))); + + // Initialize the queue we will use for processing entities into the Solr index. + DrupalQueue::get('apachesolr_indexer_entities')->createQueue(); } + /** * Implements hook_enable(). */ @@ -80,30 +84,6 @@ function apachesolr_enable() { * TODO: move all node indexing/seach code to apachesolr_search */ function apachesolr_schema() { - $schema['apachesolr_search_node'] = array( - 'description' => t('Stores a record of when a node property changed to determine if it needs indexing by Solr.'), - 'fields' => array( - 'nid' => array( - 'description' => t('The primary identifier for a node.'), - 'type' => 'int', - 'unsigned' => TRUE, - 'not null' => TRUE), - 'status' => array( - 'description' => t('Boolean indicating whether the node is published (visible to non-administrators).'), - 'type' => 'int', - 'not null' => TRUE, - 'default' => 1), - 'changed' => array( - 'description' => t('The Unix timestamp when a node property was changed.'), - 'type' => 'int', - 'not null' => TRUE, - 'default' => 0), - ), - 'indexes' => array( - 'changed' => array('changed', 'status'), - ), - 'primary key' => array('nid'), - ); $table = drupal_get_schema_unprocessed('system', 'cache'); $table['description'] = 'Cache table for apachesolr to store Luke data and indexing information.'; $schema['cache_apachesolr'] = $table; @@ -187,6 +167,79 @@ function apachesolr_schema() { 'primary key' => array('env_id', 'name'), ); + //Predefine an amount of types that get their own table + $types = array( + 'other' => 'apachesolr_index_entities', + 'node' => 'apachesolr_index_entities_node', + ); + + foreach ($types as $type) { + $schema[$type] = array( + 'description' => t('Stores a record of when an entity changed to determine if it needs indexing by Solr.'), + 'fields' => array( + 'entity_type' => array( + 'description' => t('The type of entity.'), + 'type' => 'varchar', + 'length' => 128, + 'not null' => TRUE, + ), + 'entity_id' => array( + 'description' => t('The primary identifier for an entity.'), + 'type' => 'int', + 'unsigned' => TRUE, + 'not null' => TRUE, + ), + 'bundle' => array( + 'description' => t('The bundle to which this entity belongs.'), + 'type' => 'varchar', + 'length' => 128, + 'not null' => TRUE, + ), + 'status' => array( + 'description' => t('Boolean indicating whether the entity is visible to non-administrators (eg, published for nodes).'), + 'type' => 'int', + 'not null' => TRUE, + 'default' => 1, + ), + 'changed' => array( + 'description' => t('The Unix timestamp when an entity was changed.'), + 'type' => 'int', + 'not null' => TRUE, + 'default' => 0, + ), + ), + 'indexes' => array( + 'changed' => array('changed', 'status'), + ), + 'primary key' => array('entity_id'), + ); + } + + $schema['apachesolr_index_bundles'] = array( + 'description' => t('Records what bundles we should be indexing for a given core.'), + 'fields' => array( + 'core' => array( + 'description' => t('The name of the core.'), + 'type' => 'varchar', + 'length' => 128, + 'not null' => TRUE, + ), + 'entity_type' => array( + 'description' => t('The type of entity.'), + 'type' => 'varchar', + 'length' => 128, + 'not null' => TRUE, + ), + 'bundle' => array( + 'description' => t('The bundle to index.'), + 'type' => 'varchar', + 'length' => 128, + 'not null' => TRUE, + ), + ), + 'primary key' => array('core', 'entity_type', 'bundle'), + ); + return $schema; } diff --git a/apachesolr.module b/apachesolr.module index 4616d36..18e4b48 100644 --- a/apachesolr.module +++ b/apachesolr.module @@ -10,6 +10,17 @@ define('APACHESOLR_READ_ONLY', 1); define('APACHESOLR_API_VERSION', '3.0'); /** + * Implements hook_init(). + */ +function apachesolr_init() { + module_load_include('inc', 'apachesolr', 'index'); + if (arg(0) == 'admin') { + // Add the CSS for this module + drupal_add_css(drupal_get_path('module', 'apachesolr') . '/apachesolr.css'); + } +} + +/** * Implements hook_menu(). */ function apachesolr_menu() { @@ -17,18 +28,21 @@ function apachesolr_menu() { $items['admin/config/search/apachesolr'] = array( 'title' => 'Apache Solr search', 'description' => 'Administer Apache Solr.', - 'page callback' => 'apachesolr_index_page', + 'page callback' => 'apachesolr_status_page', 'access arguments' => array('administer search'), 'weight' => -8, 'file' => 'apachesolr.admin.inc', ); $items['admin/config/search/apachesolr/index'] = array( - 'title' => 'Default Index', + 'title' => 'Default index', + 'description' => 'Administer Apache Solr.', + 'page callback' => 'apachesolr_status_page', 'access arguments' => array('administer search'), - 'weight' => -10, + 'weight' => -8, 'file' => 'apachesolr.admin.inc', 'type' => MENU_DEFAULT_LOCAL_TASK, ); + $items['admin/config/search/apachesolr/settings'] = array( 'title' => 'Settings', 'weight' => 10, @@ -40,7 +54,6 @@ function apachesolr_menu() { ); $settings_path = 'admin/config/search/apachesolr/settings/'; - $items[$settings_path . '%apachesolr_environment'] = array( 'title' => 'Apache Solr search environment edit', 'page callback' => 'apachesolr_index_page', @@ -54,9 +67,49 @@ function apachesolr_menu() { 'page callback' => 'apachesolr_index_page', 'page arguments' => array(5), 'access arguments' => array('administer search'), - 'weight' => 2, + 'weight' => 0, 'file' => 'apachesolr.admin.inc', - 'type' => MENU_LOCAL_TASK, + 'type' => MENU_DEFAULT_LOCAL_TASK, + ); + $items[$settings_path . '%apachesolr_environment/index/remaining'] = array( + 'title' => 'Remaining', + 'page callback' => 'drupal_get_form', + 'page arguments' => array('apachesolr_index_action_form_remaining_confirm'), + 'file' => 'apachesolr.admin.inc', + 'access arguments' => array('administer search'), + 'type' => MENU_CALLBACK, + ); + $items[$settings_path . '%apachesolr_environment/index/delete'] = array( + 'title' => 'Reindex', + 'page callback' => 'drupal_get_form', + 'page arguments' => array('apachesolr_index_action_form_delete_confirm'), + 'file' => 'apachesolr.admin.inc', + 'access arguments' => array('administer search'), + 'type' => MENU_CALLBACK, + ); + $items[$settings_path . '%apachesolr_environment/index/reset'] = array( + 'title' => 'Reindex', + 'page callback' => 'drupal_get_form', + 'page arguments' => array('apachesolr_index_action_form_reset_confirm'), + 'file' => 'apachesolr.admin.inc', + 'access arguments' => array('administer search'), + 'type' => MENU_CALLBACK, + ); + $items[$settings_path . '%apachesolr_environment/index/reset/confirm'] = array( + 'title' => 'Confirm the re-indexing of all content', + 'page callback' => 'drupal_get_form', + 'page arguments' => array('apachesolr_clear_index_confirm'), + 'access arguments' => array('administer search'), + 'file' => 'apachesolr.admin.inc', + 'type' => MENU_CALLBACK, + ); + $items[$settings_path . '%apachesolr_environment/index/delete/confirm'] = array( + 'title' => 'Confirm index deletion', + 'page callback' => 'drupal_get_form', + 'page arguments' => array('apachesolr_delete_index_confirm'), + 'access arguments' => array('administer search'), + 'file' => 'apachesolr.admin.inc', + 'type' => MENU_CALLBACK, ); $items[$settings_path . '%apachesolr_environment/edit'] = array( @@ -199,16 +252,6 @@ function apachesolr_enabled_facets_page($realm_name, $environment = NULL) { } /** - * Implements hook_init(). - */ -function apachesolr_init() { - if (arg(0) == 'admin') { - // Add the CSS for this module - drupal_add_css(drupal_get_path('module', 'apachesolr') . '/apachesolr.css'); - } -} - -/** * Implements hook_facetapi_searcher_info(). */ function apachesolr_facetapi_searcher_info() { @@ -269,6 +312,7 @@ function apachesolr_facetapi_query_types() { /** * Implements hook_facetapi_facet_info(). + * Currently it only supports the node entity type */ function apachesolr_facetapi_facet_info($searcher_info) { $facets = array(); @@ -558,48 +602,6 @@ function apachesolr_node_type_update($info) { } /** - * Helper function for modules implementing hook_search's 'status' op. - */ -function apachesolr_index_status($namespace) { - $excluded_types = apachesolr_get_excluded_types($namespace); - list($last_change, $last_nid) = apachesolr_get_last_change($namespace); - - $query = db_select('apachesolr_search_node', 'asn')->condition('asn.status', 1); - apachesolr_query_add_excluded_types($query, $excluded_types); - $total = $query->countQuery()->execute()->fetchField(); - - $query = db_select('apachesolr_search_node', 'asn') - ->condition('asn.status', 1) - ->condition(db_or()->condition('asn.changed', $last_change, '>')->condition(db_and()->condition('asn.changed', $last_change)->condition('asn.nid', $last_nid, '>'))); - apachesolr_query_add_excluded_types($query, $excluded_types); - $remaining = $query->countQuery()->execute()->fetchField(); - - return array('remaining' => $remaining, 'total' => $total); -} - -/** - * Returns last_changed and last_nid for an indexing namespace. - */ -function apachesolr_get_last_index($namespace) { - $stored = variable_get('apachesolr_index_last', array()); - return isset($stored[$namespace]) ? $stored[$namespace] : array('last_change' => 0, 'last_nid' => 0); -} - -/** - * Clear a specific namespace's last changed and nid, or clear all. - */ -function apachesolr_clear_last_index($namespace = '') { - if ($namespace) { - $stored = variable_get('apachesolr_index_last', array()); - unset($stored[$namespace]); - variable_set('apachesolr_index_last', $stored); - } - else { - variable_del('apachesolr_index_last'); - } -} - -/** * Truncate and rebuild the apachesolr_search_node table, reset the apachesolr_index_last variable. * This is the most complete way to force reindexing, or to build the indexing table for the * first time. @@ -608,7 +610,7 @@ function apachesolr_clear_last_index($namespace = '') { * A single content type to be reindexed, leaving the others unaltered. */ function apachesolr_rebuild_index_table($type = NULL) { - if (isset($type)) { + /*if (isset($type)) { $sel_query = db_select('node') ->fields('node', array('nid')) ->condition('type', $type); @@ -647,176 +649,10 @@ function apachesolr_rebuild_index_table($type = NULL) { // Make sure no nodes end up with a timestamp that's in the future. db_update('apachesolr_search_node')->condition('changed', REQUEST_TIME, '>')->fields(array('changed' => REQUEST_TIME))->execute(); apachesolr_clear_last_index(); - } + }*/ cache_clear_all('*', 'cache_apachesolr', TRUE); } -function apachesolr_get_excluded_types($namespace) { - $excluded_types = module_invoke_all('apachesolr_types_exclude', $namespace); - if ($excluded_types) { - $excluded_types = array_unique($excluded_types); - } - return $excluded_types; -} - -function apachesolr_query_add_excluded_types($query, $excluded_types) { - if ($excluded_types) { - $query->innerJoin('node', 'n', 'n.nid = asn.nid'); - $query->condition('n.type', $excluded_types, 'NOT IN'); - } -} - -function apachesolr_get_last_change($namespace) { - extract(apachesolr_get_last_index($namespace)); - return array($last_change, $last_nid); -} - -/** - * Returns an array of rows from a query based on an indexing namespace. - */ -function apachesolr_get_nodes_to_index($namespace, $limit) { - $rows = array(); - if (apachesolr_environment_variable_get(apachesolr_default_environment(), 'apachesolr_read_only', APACHESOLR_READ_WRITE) == APACHESOLR_READ_ONLY) { - return $rows; - } - - $excluded_types = apachesolr_get_excluded_types($namespace); - list($last_change, $last_nid) = apachesolr_get_last_change($namespace); - // TODO: remove old code - // list($excluded_types, $args, $join_sql, $exclude_sql) = apachesolr_exclude_types($namespace); - // $result = db_query_range("SELECT asn.nid, asn.changed FROM {apachesolr_search_node} asn ". $join_sql ."WHERE (asn.changed > :last_changed OR (asn.changed = :last_changed AND asn.nid > :last_nid)) AND asn.status = 1 ". $exclude_sql ."ORDER BY asn.changed ASC, asn.nid ASC", 0, $limit, $args); - $query = db_select('apachesolr_search_node', 'asn') - ->fields('asn', array('nid', 'changed')) - ->condition('asn.status', 1) - ->condition(db_or()->condition('asn.changed', $last_change, '>')->condition(db_and()->condition('asn.changed', $last_change)->condition('asn.nid', $last_nid, '>'))) - ->orderBy('asn.changed', 'ASC') - ->orderBy('asn.nid', 'ASC') - ->range(0, $limit); - apachesolr_query_add_excluded_types($query, $excluded_types); - $result = $query->execute(); - return $result; -} - -/** - * Function to handle the indexing of nodes. - * - * The calling function must supply a namespace. - * Returns FALSE if no nodes were indexed (none found or error). - */ -/** - * Handles the indexing of nodes. - * - * @param array $rows - * Each $row in $rows must have: - * $row->nid - * $row->changed - * @param string $namespace - * Usually the calling module. Is used as a clue for other modules - * when they decide whether to create extra $documents, and is used - * to track the last_index timestamp. - * @return timestamp $position - * Either a timestamp representing the last value of apachesolr_get_last_index - * to be indexed, or FALSE if indexing failed. - */ -function apachesolr_index_nodes($rows, $namespace) { - if (!$rows) { - // Nothing to do. - return FALSE; - } - - try { - // Get the $solr object - $solr = apachesolr_get_solr(); - // If there is no server available, don't continue. - if (!$solr->ping(variable_get('apachesolr_ping_timeout', 4))) { - throw new Exception(t('No Solr instance available during indexing.')); - } - } - catch (Exception $e) { - watchdog('Apache Solr', nl2br(check_plain($e->getMessage())), NULL, WATCHDOG_ERROR); - return FALSE; - } - module_load_include('inc', 'apachesolr', 'apachesolr.index'); - $documents = array(); - $old_position = apachesolr_get_last_index($namespace); - $position = $old_position; - $position['nodes_processed'] = 0; - - // Invoke hook_apachesolr_document_handlers to find out what modules build $documents - // from nodes in this namespace. - $callbacks = module_invoke_all('apachesolr_document_handlers', 'node', $namespace); - $callbacks = array_filter($callbacks, 'is_callable'); - - // Always build the content for the index as an anonynmous user. - global $user; - drupal_save_session(FALSE); - $saved_user = $user; - $user = drupal_anonymous_user(); - - foreach ($rows as $row) { - try { - // Build node. Set reset = TRUE to avoid static caching of all nodes that get indexed. - if ($node = node_load($row->nid, NULL, TRUE)) { - foreach ($callbacks as $callback) { - // The callback can either return a $document or an array of $documents. - $documents[] = $callback($node, $namespace); - } - } - // Variables to track the last item changed. - $position['last_change'] = $row->changed; - $position['last_nid'] = $row->nid; - } - catch (Exception $e) { - // Something bad happened - log the error. - watchdog('Apache Solr', 'Error constructing documents to index:
!message', array('!message' => "Node ID: {$row->nid}
" . nl2br(strip_tags($e->getMessage()))), WATCHDOG_ERROR); - } - $position['nodes_processed']++; - } - // Restore the user. - $user = $saved_user; - drupal_save_session(TRUE); - - // Flatten $documents - $tmp = array(); - apachesolr_flatten_documents_array($documents, $tmp); - $documents = $tmp; - - if (count($documents)) { - try { - watchdog('Apache Solr', 'Adding @count documents.', array('@count' => count($documents))); - // Chunk the adds by 20s - $docs_chunk = array_chunk($documents, 20); - foreach ($docs_chunk as $docs) { - $solr->addDocuments($docs); - } - // Set the timestamp to indicate an index update. - apachesolr_index_set_last_updated(REQUEST_TIME); - } - catch (Exception $e) { - $nids = array(); - if (!empty($docs)) { - foreach ($docs as $doc) { - $nids[] = $doc->entity_id; - } - } - watchdog('Apache Solr', 'Indexing failed on one of the following nodes: @nids
!message', array( - '@nids' => implode(', ', $nids), - '!message' => nl2br(strip_tags($e->getMessage())), - ), WATCHDOG_ERROR); - return FALSE; - } - } - - // Save the new position in case it changed. - if ($namespace && $position != $old_position) { - $stored = variable_get('apachesolr_index_last', array()); - $stored[$namespace] = $position; - variable_set('apachesolr_index_last', $stored); - } - - return $position; -} - /** * Convert date from timestamp into ISO 8601 format. * http://lucene.apache.org/solr/api/org/apache/solr/schema/DateField.html @@ -844,114 +680,6 @@ function apachesolr_flatten_documents_array($documents, &$tmp) { } } -function apachesolr_delete_node_from_index($node) { - $failed = &drupal_static(__FUNCTION__, FALSE); - if ($failed) { - return FALSE; - } - try { - $solr = apachesolr_get_solr(); - $solr->deleteById(apachesolr_document_id($node->nid)); - apachesolr_index_set_last_updated(REQUEST_TIME); - return TRUE; - } - catch (Exception $e) { - watchdog('Apache Solr', nl2br(check_plain($e->getMessage())), NULL, WATCHDOG_ERROR); - // Don't keep trying queries if they are failing. - $failed = TRUE; - return FALSE; - } -} - -/** - * Set the timestamp of the last index update - * @param $updated - * A timestamp or zero. If zero, the variable is deleted. - */ -function apachesolr_index_set_last_updated($updated = 0) { - if ($updated) { - variable_set('apachesolr_index_updated', (int) $updated); - } - else { - variable_del('apachesolr_index_updated'); - } -} - -/** - * Get the timestamp of the last index update. - * @return integer (timestamp) - */ -function apachesolr_index_get_last_updated() { - return variable_get('apachesolr_index_updated', 0); -} - -/** - * Implements hook_cron(). - */ -function apachesolr_cron() { - - // Indexes in read-only mode do not change the index, so will not update, delete, or optimize during cron. - if (apachesolr_environment_variable_get(apachesolr_default_environment(), 'apachesolr_read_only', APACHESOLR_READ_WRITE) == APACHESOLR_READ_ONLY) { - return; - } - - // Mass update and delete functions are in the include file. - module_load_include('inc', 'apachesolr', 'apachesolr.index'); - apachesolr_cron_check_node_table(); - try { - $solr = apachesolr_get_solr(); - - // Check for unpublished content that wasn't deleted from the index. - $query = db_select('apachesolr_search_node', 'asn'); - $n_alias = $query->innerJoin('node', 'n', 'n.nid = asn.nid'); - $query->fields($n_alias, array('nid', 'status')); - $query->where("asn.status <> $n_alias.status"); - $query->addTag('apachesolr_cron_update'); - $result = $query->execute(); - foreach ($result as $node) { - apachesolr_node_update($node, FALSE); - } - - // Check for deleted content that wasn't deleted from the index. - $query = db_select('apachesolr_search_node', 'asn'); - $query->fields('asn', array('nid')); - $n_alias = $query->leftJoin('node', 'n', 'n.nid = asn.nid'); - $query->isNull("$n_alias.nid"); - $query->addTag('apachesolr_cron_delete'); - $result = $query->execute(); - foreach ($result as $node) { - apachesolr_node_delete($node, FALSE); - } - - // Optimize the index (by default once a day). - $optimize_interval = variable_get('apachesolr_optimize_interval', 60 * 60 * 24); - $last = variable_get('apachesolr_last_optimize', 0); - $time = REQUEST_TIME; - if ($optimize_interval && ($time - $last > $optimize_interval)) { - $solr->optimize(FALSE, FALSE); - variable_set('apachesolr_last_optimize', $time); - apachesolr_index_set_last_updated($time); - } - - // Only clear the cache if the index changed. - // TODO: clear on some schedule if running multi-site. - $updated = apachesolr_index_get_last_updated(); - if ($updated) { - $solr->clearCache(); - // Re-populate the luke cache. - $solr->getLuke(); - // TODO: an admin interface for setting this. Assume for now 5 minutes. - if ($time - $updated >= variable_get('apachesolr_cache_delay', 300)) { - // Clear the updated flag. - apachesolr_index_set_last_updated(0); - } - } - } - catch (Exception $e) { - watchdog('Apache Solr', nl2br(check_plain($e->getMessage())) . ' in apachesolr_cron', NULL, WATCHDOG_ERROR); - } -} - /** * Implements hook_flush_caches(). */ @@ -979,47 +707,6 @@ function apachesolr_clear_cache($env_id) { } /** - * Implements hook_node_insert(). - */ -function apachesolr_node_insert($node) { - // Make sure no node ends up with a timestamp that's in the future by using - // REQUEST_TIME rather than the node's changed or created timestamp. - db_insert('apachesolr_search_node')->fields(array('nid' => $node->nid, 'status' => $node->status, 'changed' => REQUEST_TIME))->execute(); -} - -/** - * Implements hook_node_delete(). - */ -function apachesolr_node_delete($node, $set_message = TRUE) { - if (apachesolr_delete_node_from_index($node)) { - // There was no exception, so delete from the table. - db_delete('apachesolr_search_node')->condition('nid', $node->nid)->execute(); - if ($set_message && user_access('administer search') && variable_get('apachesolr_set_nodeapi_messages', 1)) { - apachesolr_set_stats_message('Deleted content will be removed from the Apache Solr search index in approximately @autocommit_time.'); - } - } -} - -/** - * Implements hook_node_update(). - */ -function apachesolr_node_update($node, $set_message = TRUE) { - // Check if the node has gone from published to unpublished. - if (!$node->status && db_select('apachesolr_search_node', 'asn')->fields('asn', array('status'))->condition('nid', $node->nid)->execute()) { - if (apachesolr_delete_node_from_index($node)) { - // There was no exception, so update the table. - db_update('apachesolr_search_node')->condition('nid', $node->nid)->fields(array('changed' => REQUEST_TIME, 'status' => $node->status))->execute(); - if ($set_message && user_access('administer search') && variable_get('apachesolr_set_nodeapi_messages', 1)) { - apachesolr_set_stats_message('Unpublished content will be removed from the Apache Solr search index in approximately @autocommit_time.'); - } - } - } - else { - db_update('apachesolr_search_node')->condition('nid', $node->nid)->fields(array('changed' => REQUEST_TIME, 'status' => $node->status))->execute(); - } -} - -/** * Call drupal_set_message() with the text. * * The text is translated with t() and substituted using Solr stats. @@ -1097,6 +784,80 @@ function apachesolr_block_view($delta = '') { } /** + * Implements hook_cron(). + */ +function apachesolr_cron() { + // Indexes in read-only mode do not change the index, so will not update, delete, or optimize during cron. + if (apachesolr_environment_variable_get(apachesolr_default_environment(), 'apachesolr_read_only', APACHESOLR_READ_WRITE) == APACHESOLR_READ_ONLY) { + return; + } + + apachesolr_cron_check_node_table(); + try { + $solr = apachesolr_get_solr(); + + // Check for unpublished content that wasn't deleted from the index. + $query = db_select('apachesolr_search_node', 'asn'); + $n_alias = $query->innerJoin('node', 'n', 'n.nid = asn.nid'); + $query->fields($n_alias, array('nid', 'status')); + $query->where("asn.status <> $n_alias.status"); + $query->addTag('apachesolr_cron_update'); + $result = $query->execute(); + foreach ($result as $node) { + apachesolr_node_update($node, FALSE); + } + + // Check for deleted content that wasn't deleted from the index. + $query = db_select('apachesolr_search_node', 'asn'); + $query->fields('asn', array('nid')); + $n_alias = $query->leftJoin('node', 'n', 'n.nid = asn.nid'); + $query->isNull("$n_alias.nid"); + $query->addTag('apachesolr_cron_delete'); + $result = $query->execute(); + foreach ($result as $node) { + apachesolr_node_delete($node, FALSE); + } + + // Optimize the index (by default once a day). + $optimize_interval = variable_get('apachesolr_optimize_interval', 60 * 60 * 24); + $last = variable_get('apachesolr_last_optimize', 0); + $time = REQUEST_TIME; + if ($optimize_interval && ($time - $last > $optimize_interval)) { + $solr->optimize(FALSE, FALSE); + variable_set('apachesolr_last_optimize', $time); + apachesolr_index_set_last_updated($time); + } + + // Only clear the cache if the index changed. + // TODO: clear on some schedule if running multi-site. + $updated = apachesolr_index_get_last_updated(); + if ($updated) { + $solr->clearCache(); + // Re-populate the luke cache. + $solr->getLuke(); + // TODO: an admin interface for setting this. Assume for now 5 minutes. + if ($time - $updated >= variable_get('apachesolr_cache_delay', 300)) { + // Clear the updated flag. + apachesolr_index_set_last_updated(0); + } + } + } + catch (Exception $e) { + watchdog('Apache Solr', nl2br(check_plain($e->getMessage())) . ' in apachesolr_cron', NULL, WATCHDOG_ERROR); + } + + $cron_limit = variable_get('apachesolr_cron_limit', 50); + $rows = apachesolr_index_get_entities_to_index($cron_limit); + $documents = array(); + foreach ($rows as $row) { + $documents = array_merge($documents, apachesolr_index_index_entity($row)); + } + apachesolr_index_send_to_solr($documents); + apachesolr_index_set_last_updated(REQUEST_TIME); +} + + +/** * Implements hook_form_[form_id]_alter(). * * Make sure to flush cache when content types are changed. @@ -1881,6 +1642,98 @@ function apachesolr_facet_form_validate($form, &$form_state) { } /** + * Implements hook_entity_info_alter(). + */ +function apachesolr_entity_info_alter(&$entity_info) { + module_load_include('inc', 'apachesolr', 'apachesolr.index'); + $default_info = apachesolr_index_get_entity_defaults(); + // First set defaults so that we needn't worry about NULL keys. + foreach (array_keys($entity_info) as $type) { + $entity_info[$type]['apachesolr'] = array(); + if (isset($default_info[$type])) { + $entity_info[$type]['apachesolr'] += $default_info[$type]; + } + $default = array( + 'indexable' => FALSE, + 'status callback' => '', + 'document callback' => '', + 'reindex callback' => '', + ); + $entity_info[$type]['apachesolr'] += $default; + } + + // For any supported entity type and bundle, flag it for indexing. + foreach ($entity_info as $entity_type => $info) { + if ($info['apachesolr']['indexable']) { + $supported = apachesolr_index_get_bundles('default', $entity_type); + foreach (array_keys($info['bundles']) as $bundle) { + if (in_array($bundle, $supported)) { + $entity_info[$entity_type]['bundles'][$bundle]['apachesolr']['index'] = TRUE; + } + } + } + } +} + +/** + * Implements hook_entity_insert(). + */ +function apachesolr_entity_insert($entity, $type) { + // For our purposes there's really no difference between insert and update. + return apachesolr_index_entity_update($entity, $type); +} + +/** + * Implements hook_entity_update(). + */ +function apachesolr_entity_update($entity, $type) { + + if (apachesolr_entity_should_index($entity, $type)) { + $info = entity_get_info($type); + list($id, $vid, $bundle) = entity_extract_ids($type, $entity); + + $status_callback = apachesolr_entity_get_callback($type, 'status callback'); + $status = 0; + if (is_callable($status_callback)) { + $status = $status_callback($entity, $type); + } + + $indexer_table = apachesolr_index_get_indexer_table($type); + + // If we haven't seen this entity before it may not be there, so merge + // instead of update. + db_merge($indexer_table) + ->key(array( + 'entity_type' => $type, + 'entity_id' => $id, + )) + ->fields(array( + 'bundle' => $bundle, + 'status' => $status, + 'changed' => REQUEST_TIME, + )) + ->execute(); + } +} + +/** + * Implements hook_entity_delete(). + * + * @see apachesolr_node_delete(). + */ +function apachesolr_entity_delete($entity, $type) { + if (apachesolr_index_delete_entity_from_index($type, $entity)) { + // There was no exception, so delete from the table. + list($id) = entity_extract_ids($type, $entity); + $indexer_table = apachesolr_index_get_indexer_table($type); + db_delete($indexer_table) + ->condition('entity_type', $type) + ->condition('entity_id', $id) + ->execute(); + } +} + +/** * Returns array containing information about node fields that should be indexed */ function apachesolr_entity_fields($entity_type = 'node') { @@ -1909,7 +1762,7 @@ function apachesolr_entity_fields($entity_type = 'node') { } // Allow other modules to add or alter mappings. - drupal_alter('apachesolr_field_mappings', $mappings); + drupal_alter('apachesolr_field_mappings', $mappings, $entity_type); $modules = system_get_info('module'); $instances = field_info_instances($entity_type); foreach (field_info_fields() as $field_name => $field) { @@ -1955,6 +1808,47 @@ function apachesolr_entity_fields($entity_type = 'node') { } /** + * Implements hook_apachesolr_index_document_build(). + */ +function field_apachesolr_index_document_build(ApacheSolrDocument $document, $entity, $entity_type) { + $info = entity_get_info($entity_type); + if ($info['fieldable']) { + // Handle fields including taxonomy. + $indexed_fields = apachesolr_entity_fields($entity_type); + foreach ($indexed_fields as $index_key => $field_info) { + $field_name = $field_info['field']['field_name']; + // See if the node has fields that can be indexed + if (isset($entity->{$field_name})) { + // Got a field. + $function = $field_info['indexing_callback']; + if ($function && function_exists($function)) { + // NOTE: This function should always return an array. One + // entity field may be indexed to multiple Solr fields. + $fields = $function($entity, $field_name, $index_key, $field_info); + foreach ($fields as $field) { + // It's fine to use this method also for single value fields. + $document->setMultiValue($field['key'], $field['value']); + } + } + } + } + } +} + +/** + * Implements hook_apachesolr_index_document_node_book_build(). + * + * Adds book module support + */ +function apachesolr_apachesolr_index_document_node_book_build(ApacheSolrDocument $document, $entity, $entity_type) { + // Index book module data. + if (!empty($entity->book['bid'])) { + // Hard-coded - must change if apachesolr_index_key() changes. + $document->is_book_bid = (int) $entity->book['bid']; + } +} + +/** * Strip html tags and also control characters that cause Jetty/Solr to fail. */ function apachesolr_clean_text($text) { @@ -2095,36 +1989,10 @@ function apachesolr_entity_get_callback($entity_type, $callback, $bundle = NULL) else { $callback_function = NULL; } - return $callback_function; } /** - * Determines if we should index the provided entity. - * - * Whether or not a given entity is indexed is determined on a per-bundle basis. - * Entities/Bundles that have no index flag are presumed to not get indexed. - * - * @param stdClass $entity - * The entity we may or may not want to index. - * @param string $type - * The type of entity. - * @return boolean - * TRUE if this entity should be indexed, FALSE otherwise. - */ -function apachesolr_entity_should_index($entity, $type) { - $info = entity_get_info($type); - list($id, $vid, $bundle) = entity_extract_ids($type, $entity); - - if ($bundle && isset($info['bundles'][$bundle]['apachesolr']['index']) && $info['bundles'][$bundle]['apachesolr']['index']) { - return TRUE; - } - - return FALSE; -} - - -/** * Implements hook_theme(). */ function apachesolr_theme() { @@ -2184,6 +2052,29 @@ function apachesolr_hook_info() { ) ); + $hooks['apachesolr_index_document_build'] = array( + 'group' => 'apachesolr', + ); + $hooks['apachesolr_index_document_alter'] = array( + 'group' => 'apachesolr', + ); + foreach (entity_get_info() as $entity_type => $info) { + $hooks['apachesolr_index_document_' . $entity_type . '_build'] = array( + 'group' => 'apachesolr', + ); + $hooks['apachesolr_index_document_' . $entity_type . '_alter'] = array( + 'group' => 'apachesolr', + ); + foreach ($info['bundles'] as $bundle => $bundle_info) { + $hooks['apachesolr_index_document_' . $entity_type . '_' . $bundle . '_build'] = array( + 'group' => 'apachesolr', + ); + $hooks['apachesolr_index_document_' . $entity_type . '_' . $bundle . '_alter'] = array( + 'group' => 'apachesolr', + ); + } + } + return $hooks; } diff --git a/apachesolr_search.admin.inc b/apachesolr_search.admin.inc index aacefd7..cbc7661 100644 --- a/apachesolr_search.admin.inc +++ b/apachesolr_search.admin.inc @@ -846,60 +846,9 @@ function apachesolr_search_type_boost_form($env_id) { '#default_value' => isset($type_boosts[$type]) ? $type_boosts[$type] : 0, ); } - - $form['type_boost']['apachesolr_search_excluded_types'] = array( - '#type' => 'checkboxes', - '#title' => t('Types to exclude from the search index'), - '#options' => $names, - '#default_value' => apachesolr_environment_variable_get($env_id, 'apachesolr_search_excluded_types', array()), - '#description' => t("Specify here which node types should be totally excluded from the search index. Content excluded from the index will never appear in any search results."), - ); - $form['#original_excluded_types'] = $form['type_boost']['apachesolr_search_excluded_types']['#default_value']; - - $form['#submit'][] = 'apachesolr_search_type_boost_form_submit'; - return $form; } -/** - * Submit callback for apachesolr_search_type_boost_form(). - * - * This is called before system_settings_form_submit(). - */ -function apachesolr_search_type_boost_form_submit($form, &$form_state) { - $old_excluded_types = $form['#original_excluded_types']; - $new_excluded_types = $form_state['values']['apachesolr_search_excluded_types']; - // Check whether we are resetting the values. - if ($form_state['clicked_button']['#value'] == t('Reset to defaults')) { - $new_excluded_types = array(); - } - - foreach ($new_excluded_types as $type => $excluded) { - // Remove newly omitted node types. - // Note - we omit a check on empty($old_excluded_types[$type]) so that - // the admin can re-submit this page if the delete operation fails. - if (!empty($new_excluded_types[$type])) { - try { - $solr = apachesolr_get_solr($form['#env_id']); - $solr->deleteByQuery("bundle:$type"); - apachesolr_index_set_last_updated(REQUEST_TIME); - } - catch (Exception $e) { - watchdog('Apache Solr', nl2br(check_plain($e->getMessage())), NULL, WATCHDOG_ERROR); - drupal_set_message(t('The Apache Solr search engine is not available. Please contact your site administrator.'), 'error'); - } - } - } - - foreach ($old_excluded_types as $type => $excluded) { - // Set no longer omitted node types for reindexing. - if (empty($new_excluded_types[$type]) && !empty($old_excluded_types[$type])) { - $nids = db_select('node')->fields('node', array('nid'))->condition('type', $type); - db_update('apachesolr_search_node')->fields(array('changed' => REQUEST_TIME))->condition('nid', $nids, 'IN')->execute(); - } - } -} - /** * MoreLikeThis administration and utility functions. diff --git a/apachesolr_search.module b/apachesolr_search.module index a81adc6..f51c053 100644 --- a/apachesolr_search.module +++ b/apachesolr_search.module @@ -286,7 +286,6 @@ function apachesolr_search_form_block_admin_display_form_alter(&$form) { * Implements hook_block_configure(). */ function apachesolr_search_block_configure($delta = '') { - dsm($delta); if ($delta != 'sort') { require_once(drupal_get_path('module', 'apachesolr') . '/apachesolr_search.admin.inc'); return apachesolr_search_mlt_block_form($delta); @@ -409,15 +408,6 @@ function theme_apachesolr_search_mlt_recommendation_block($vars) { } /** - * Implements hook_cron(). Indexes nodes. - */ -function apachesolr_search_cron() { - $cron_limit = variable_get('apachesolr_cron_limit', 50); - $rows = apachesolr_get_nodes_to_index('apachesolr_search', $cron_limit); - apachesolr_index_nodes($rows, 'apachesolr_search'); -} - -/** * Implements hook_apachesolr_types_exclude(). */ function apachesolr_search_apachesolr_types_exclude($namespace) { @@ -453,14 +443,16 @@ function apachesolr_search_search_info() { * Implementation of hook_search_reset(). */ function apachesolr_search_search_reset() { - apachesolr_clear_last_index('apachesolr_search'); + module_load_include('inc', 'apachesolr', 'apachesolr.index'); + apachesolr_index_mark_all_for_reindex(); } /** * Implementation of hook_search_status(). */ function apachesolr_search_search_status() { - return apachesolr_index_status('apachesolr_search'); + module_load_include('inc', 'apachesolr', 'apachesolr.index'); + return apachesolr_index_status(); } /**