diff --git a/README.txt b/README.txt index 63fe32b..a03a256 100644 --- a/README.txt +++ b/README.txt @@ -121,10 +121,18 @@ indexed again. You can also select a server that is at the moment disabled, or choose to let the index lie on no server at all, for the time being. Note, however, that you can only create enabled indexes on an enabled server. Also, disabling a server will disable all indexes that lie on it. -Lastly, the cron limit option allows you to set whether, and how many, items -will be indexed for this index when cron runs (and the index is enabled). Items -can also be indexed manually, so even if this is set to 0, the index can still -be used. +The "Index items immediately" option specifies that you want items to be +directly re-indexed after being changed, instead of waiting for the next cron +run. Use this if it is important that users see no stale data in searches, and +only when your setup enables relatively fast indexing. +Lastly, the "Cron batch size" option allows you to set whether items will be +indexed when cron runs (as long as the index is enabled), and how many items +will be indexed in a single batch. The best value for this setting depends on +how time-consuming indexing is for your setup, which in turn depends mostly on +the server used and the enabled data alterations. You should set it to a number +of items which can easily be indexed in 10 seconds' time. Items can also be +indexed manually, or directly when they are changed, so even if this is set to +0, the index can still be used. - Indexed fields (Configuration > Search API > [Index name] > Fields) @@ -160,7 +168,7 @@ On this page you can view how much of the entities are already indexed and also control indexing. With the "Index now" button (displayed only when there are still unindexed items) you can directly index a certain number of "dirty" items (i.e., items not yet indexed in their current state). Setting "-1" as the number -will index all of those items, similar to the cron limit setting. +will index all of those items, similar to the cron batch size setting. When you change settings that could affect indexing, and the index is not automatically marked for re-indexing, you can do this manually with the "Re-index content" button. All items in the index will be marked as dirty and be diff --git a/includes/datasource.inc b/includes/datasource.inc index 3b702ea..7adf111 100644 --- a/includes/datasource.inc +++ b/includes/datasource.inc @@ -144,16 +144,38 @@ interface SearchApiDataSourceControllerInterface { /** * Set the tracking status of the given items to "changed"/"dirty". * + * Unless $dequeue is set to TRUE, this operation is ignored for items whose + * status is not "indexed". + * * @param $item_ids * Either an array with the IDs of the changed items. Or FALSE to mark all * items as changed for the given indexes. * @param array $indexes * The indexes for which the change should be tracked. + * @param $dequeue + * If set to TRUE, also change the status of queued items. + * + * @throws SearchApiDataSourceException + * If any of the indexes doesn't use the same item type as this controller. + */ + public function trackItemChange($item_ids, array $indexes, $dequeue = FALSE); + + /** + * Set the tracking status of the given items to "queued". + * + * Queued items are not marked as "dirty" even when they are changed, and they + * are not returned by the getChangedItems() method. + * + * @param $item_ids + * Either an array with the IDs of the queued items. Or FALSE to mark all + * items as queued for the given indexes. + * @param SearchApiIndex $index + * The index for which the items were queued. * * @throws SearchApiDataSourceException * If any of the indexes doesn't use the same item type as this controller. */ - public function trackItemChange($item_ids, array $indexes); + public function trackItemQueued($item_ids, SearchApiIndex $index); /** * Set the tracking status of the given items to "indexed". @@ -477,16 +499,21 @@ abstract class SearchApiAbstractDataSourceController implements SearchApiDataSou /** * Set the tracking status of the given items to "changed"/"dirty". * + * Unless $dequeue is set to TRUE, this operation is ignored for items whose + * status is not "indexed". + * * @param $item_ids * Either an array with the IDs of the changed items. Or FALSE to mark all * items as changed for the given indexes. * @param array $indexes * The indexes for which the change should be tracked. + * @param $dequeue + * If set to TRUE, also change the status of queued items. * * @throws SearchApiDataSourceException * If any of the indexes doesn't use the same item type as this controller. */ - public function trackItemChange($item_ids, array $indexes) { + public function trackItemChange($item_ids, array $indexes, $dequeue = FALSE) { // Types that set "track index status" to FALSE should override this method // in their data source controller and provide their own logic, if possible. if (!$this->table) { @@ -502,8 +529,38 @@ abstract class SearchApiAbstractDataSourceController implements SearchApiDataSou $this->changedColumn => REQUEST_TIME, )) ->condition($this->indexIdColumn, $index_ids, 'IN') - ->condition($this->changedColumn, 0); - if (is_array($item_ids)) { + ->condition($this->changedColumn, 0, $dequeue ? '<=' : '='); + if ($item_ids !== FALSE) { + $update->condition($this->itemIdColumn, $item_ids, 'IN'); + } + $update->execute(); + } + + /** + * Set the tracking status of the given items to "queued". + * + * Queued items are not marked as "dirty" even when they are changed, and they + * are not returned by the getChangedItems() method. + * + * @param $item_ids + * Either an array with the IDs of the queued items. Or FALSE to mark all + * items as queued for the given indexes. + * @param SearchApiIndex $index + * The index for which the items were queued. + * + * @throws SearchApiDataSourceException + * If any of the indexes doesn't use the same item type as this controller. + */ + public function trackItemQueued($item_ids, SearchApiIndex $index) { + if (!$this->table) { + return; + } + $update = db_update($this->table) + ->fields(array( + $this->changedColumn => -1, + )) + ->condition($this->indexIdColumn, $index->id); + if ($item_ids !== FALSE) { $update->condition($this->itemIdColumn, $item_ids, 'IN'); } $update->execute(); @@ -586,7 +643,7 @@ abstract class SearchApiAbstractDataSourceController implements SearchApiDataSou $select = db_select($this->table, 't'); $select->addField('t', 'item_id'); $select->condition($this->indexIdColumn, $index->id); - $select->condition($this->changedColumn, 0, '<>'); + $select->condition($this->changedColumn, 0, '>'); $select->orderBy($this->itemIdColumn, 'ASC'); if ($limit > 0) { $select->range(0, $limit); diff --git a/includes/index_entity.inc b/includes/index_entity.inc index 08630ea..ae37424 100644 --- a/includes/index_entity.inc +++ b/includes/index_entity.inc @@ -102,7 +102,7 @@ class SearchApiIndex extends Entity { /** * An array of options for configuring this index. The layout is as follows: - * - cron_limit: The maximum number of items to be indexed per cron run. + * - cron_limit: The maximum number of items to be indexed per cron batch. * - index_directly: Boolean setting whether entities are indexed immediately * after they are created or updated. * - fields: An array of all known fields for this index. Keys are the field diff --git a/search_api.admin.inc b/search_api.admin.inc index 14bb0ca..fd68678 100644 --- a/search_api.admin.inc +++ b/search_api.admin.inc @@ -573,8 +573,8 @@ function search_api_admin_add_index(array $form, array &$form_state) { ); $form['options']['cron_limit'] = array( '#type' => 'textfield', - '#title' => t('Cron limit'), - '#description' => t('Set how many items will be indexed at most during each run of cron. ' . + '#title' => t('Cron batch size'), + '#description' => t('Set how many items will be indexed at once when indexing items during a cron run. ' . '"0" means that no items will be indexed by cron for this index, "-1" means that cron should index all items at once.'), '#default_value' => SEARCH_API_DEFAULT_CRON_LIMIT, '#size' => 4, @@ -601,7 +601,7 @@ function search_api_admin_add_index_validate(array $form, array &$form_state) { $cron_limit = $form_state['values']['options']['cron_limit']; if ($cron_limit != '' . ((int) $cron_limit)) { // We don't enforce stricter rules and treat all negative values as -1. - form_set_error('options[cron_limit]', t('The cron limit must be an integer.')); + form_set_error('options[cron_limit]', t('The cron batch size must be an integer.')); } } @@ -734,7 +734,7 @@ function theme_search_api_index(array $variables) { if (!$read_only && !empty($options)) { $output .= '
' . t('Index options') . '
' . "\n"; $output .= '
' . "\n"; - $output .= '
' . t('Cron limit') . '
' . "\n"; + $output .= '
' . t('Cron batch size') . '
' . "\n"; if (empty($options['cron_limit'])) { $output .= '
' . t("Don't index during cron runs") . '
' . "\n"; } @@ -742,7 +742,7 @@ function theme_search_api_index(array $variables) { $output .= '
' . t('Unlimited') . '
' . "\n"; } else { - $output .= '
' . format_plural($options['cron_limit'], '1 item per cron run.', '@count items per cron run.') . '
' . "\n"; + $output .= '
' . format_plural($options['cron_limit'], '1 item per cron batch.', '@count items per cron batch.') . '
' . "\n"; } if (!empty($options['fields'])) { @@ -1041,8 +1041,8 @@ function search_api_admin_index_edit(array $form, array &$form_state, SearchApiI ); $form['options']['cron_limit'] = array( '#type' => 'textfield', - '#title' => t('Cron limit'), - '#description' => t('Set how many items will be indexed at most during each run of cron. ' . + '#title' => t('Cron batch size'), + '#description' => t('Set how many items will be indexed at once when indexing items during a cron run. ' . '"0" means that no items will be indexed by cron for this index, "-1" means that cron should index all items at once.'), '#default_value' => isset($index->options['cron_limit']) ? $index->options['cron_limit'] : SEARCH_API_DEFAULT_CRON_LIMIT, '#size' => 4, diff --git a/search_api.drush.inc b/search_api.drush.inc index dfc5304..fe8bcf5 100644 --- a/search_api.drush.inc +++ b/search_api.drush.inc @@ -47,7 +47,7 @@ function search_api_drush_command() { ), 'arguments' => array( 'index_id' => dt('The numeric ID or machine name of an index.'), - 'limit' => dt("The number of items to index. Use 0 to index all items. Defaults to the index's cron limit."), + 'limit' => dt("The number of items to index. Use 0 to index all items. Defaults to the index's cron batch size."), ), 'aliases' => array('sapi-i'), ); diff --git a/search_api.install b/search_api.install index 285bd7f..9b68ca8 100644 --- a/search_api.install +++ b/search_api.install @@ -178,7 +178,7 @@ function search_api_schema() { 'changed' => array( 'description' => 'Either a flag or a timestamp to indicate if or when the item was changed since it was last indexed.', 'type' => 'int', - 'unsigned' => TRUE, + 'size' => 'big', 'not null' => TRUE, 'default' => 1, ), @@ -837,3 +837,17 @@ function search_api_update_7110() { // Clear entity info caches. cache_clear_all('*', 'cache', TRUE); } + +/** + * Change the definition of the {search_api_item}.changed field. + */ +function search_api_update_7111() { + $spec = array( + 'description' => 'Either a flag or a timestamp to indicate if or when the item was changed since it was last indexed.', + 'type' => 'int', + 'size' => 'big', + 'not null' => TRUE, + 'default' => 1, + ); + db_change_field('search_api_item', 'changed', 'changed', $spec); +} diff --git a/search_api.module b/search_api.module index 102fc99..9594302 100644 --- a/search_api.module +++ b/search_api.module @@ -1,7 +1,7 @@ TRUE, 'read_only' => 0)) as $index) { $limit = isset($index->options['cron_limit']) ? $index->options['cron_limit'] : SEARCH_API_DEFAULT_CRON_LIMIT; if ($limit) { try { - $num = search_api_index_items($index, $limit); - if ($num) { - watchdog('search_api', t('Indexed !num items for index !name', array('!num' => $num, '!name' => $index->name)), NULL, WATCHDOG_INFO); + $task = array('index' => $index->machine_name); + $ids = search_api_get_items_to_index($index, -1); + if (!$ids) { + continue; } + $batches = $limit > 0 ? array_chunk($ids, $limit, TRUE) : array($ids); + foreach ($batches as $batch) { + $task['items'] = $batch; + $queue->createItem($task); + } + // Mark items as queued so they won't be inserted into the queue again + // on the next cron run. + search_api_track_item_queued($index, $ids); } catch (SearchApiException $e) { watchdog('search_api', $e->getMessage(), NULL, WATCHDOG_WARNING); @@ -237,6 +247,19 @@ function search_api_cron() { } /** + * Implements hook_cron_queue_info(). + * + * Defines a queue for saved searches that should be checked for new items. + */ +function search_api_cron_queue_info() { + return array( + 'search_api_indexing_queue' => array( + 'worker callback' => '_search_api_indexing_queue_process', + ), + ); +} + +/** * Implements hook_entity_info(). */ function search_api_entity_info() { @@ -783,6 +806,18 @@ function search_api_track_item_change($type, array $item_ids) { } /** + * Marks items as queued for indexing for the specified index. + * + * @param SearchApiIndex $index + * The index on which items were queued. + * @param array $item_ids + * The ids of the queued items. + */ +function search_api_track_item_queued(SearchApiIndex $index, array $item_ids) { + $index->datasource()->trackItemQueued($item_ids, $index); +} + +/** * Marks items as successfully indexed for the specified index. * * @param SearchApiIndex $index @@ -862,7 +897,7 @@ function search_api_index_items(SearchApiIndex $index, $limit = -1) { */ function search_api_index_specific_items(SearchApiIndex $index, array $ids) { $items = $index->loadItems($ids); - $indexed = $index->index($items); + $indexed = $items ? $index->index($items) : array(); if ($indexed) { search_api_track_item_indexed($index, $indexed); } @@ -1685,7 +1720,7 @@ function search_api_index_reindex($id) { * The index whose items should be re-indexed. */ function _search_api_index_reindex(SearchApiIndex $index) { - $index->datasource()->trackItemChange(FALSE, array($index)); + $index->datasource()->trackItemChange(FALSE, array($index), TRUE); } /** @@ -1721,6 +1756,33 @@ function search_api_index_delete($id) { } /** + * Cron queue worker callback for indexing some items. + * + * @param array $task + * An associative array containing: + * - index: The ID of the index on which items should be indexed. + * - items: The items that should be indexed. + */ +function _search_api_indexing_queue_process(array $task) { + $index = search_api_index_load($task['index']); + if ($index && $index->enabled && !$index->read_only && $task['items']) { + $indexed = search_api_index_specific_items($index, $task['items']); + $num = count($indexed); + // If some items couldn't be indexed, mark them as dirty again. + if ($num < count($task['items'])) { + // Believe it or not but this is actually quite faster than the equivalent + // $diff = array_diff($task['items'], $indexed); + $diff = array_keys(array_diff_key(array_flip($task['items']), array_flip($indexed))); + // Mark the items as dirty again. + $index->datasource()->trackItemChange($diff, array($index), TRUE); + } + if ($num) { + watchdog('search_api', t('Indexed !num items for index !name', array('!num' => $num, '!name' => $index->name)), NULL, WATCHDOG_INFO); + } + } +} + +/** * Helper function to be used as a "property info alter" callback. * * If a wrapped entity is passed to this function, all its available properties diff --git a/search_api.test b/search_api.test index 8f82bb3..f0923ad 100644 --- a/search_api.test +++ b/search_api.test @@ -139,7 +139,7 @@ class SearchApiWebTest extends DrupalWebTestCase { $this->assertFalse($index->enabled, t('Status correctly inserted.')); $this->assertEqual($index->description, $values['description'], t('Description correctly inserted.')); $this->assertNull($index->server, t('Index server correctly inserted.')); - $this->assertEqual($index->options['cron_limit'], $values['options[cron_limit]'], t('Cron limit correctly inserted.')); + $this->assertEqual($index->options['cron_limit'], $values['options[cron_limit]'], t('Cron batch size correctly inserted.')); $values = array( 'additional[field]' => 'parent', @@ -213,7 +213,7 @@ class SearchApiWebTest extends DrupalWebTestCase { $this->assertTitle('Search API test index | Drupal', t('Correct title when viewing index.')); $this->assertText('An index used for testing.', t('!field displayed.', array('!field' => t('Description')))); $this->assertText('Search API test entity', t('!field displayed.', array('!field' => t('Item type')))); - $this->assertText(format_plural(5, '1 item per cron run.', '@count items per cron run.'), t('!field displayed.', array('!field' => t('Cron limit')))); + $this->assertText(format_plural(5, '1 item per cron batch.', '@count items per cron batch.'), t('!field displayed.', array('!field' => t('Cron batch size')))); $this->drupalGet("admin/config/search/search_api/index/$id/status"); $this->assertText(t('The index is currently disabled.'), t('"Disabled" status displayed.'));