diff --git a/README.txt b/README.txt index 9ea75ed..6ea6252 100644 --- a/README.txt +++ b/README.txt @@ -210,12 +210,6 @@ search_api_index_worker_callback_runtime: API will spend indexing (for all indexes combined) in each cron run. The default is 15 seconds. -search_api_batch_per_cron: - By changing this variable, you can define how many batch items are created on - a single cron run. The value is per index, so on a site with 5 indexes with a - cron limit of 100 each, the default value of 10 will load and queue up to 5000 - search items in up to 50 batch items. - Information for developers -------------------------- diff --git a/includes/datasource.inc b/includes/datasource.inc index ba0d2ba..a515d66 100644 --- a/includes/datasource.inc +++ b/includes/datasource.inc @@ -160,7 +160,9 @@ interface SearchApiDataSourceControllerInterface { * @param array $indexes * The indexes for which the change should be tracked. * @param $dequeue - * If set to TRUE, also change the status of queued items. + * (deprecated) If set to TRUE, also change the status of queued items. + * The concept of queued items will be removed in the Drupal 8 version of + * this module. * * @throws SearchApiDataSourceException * If any of the indexes doesn't use the same item type as this controller. @@ -180,7 +182,12 @@ interface SearchApiDataSourceControllerInterface { * The index for which the items were queued. * * @throws SearchApiDataSourceException - * If any of the indexes doesn't use the same item type as this controller. + * If the index doesn't use the same item type as this controller. + * + * @deprecated + * As of Search API 1.10, the cron queue is not used for indexing anymore, + * therefore this method has become useless. It will be removed in the + * Drupal 8 version of this module. */ public function trackItemQueued($item_ids, SearchApiIndex $index); @@ -189,7 +196,7 @@ interface SearchApiDataSourceControllerInterface { * * @param array $item_ids * The IDs of the indexed items. - * @param SearchApiIndex $indexes + * @param SearchApiIndex $index * The index on which the items were indexed. * * @throws SearchApiDataSourceException @@ -571,21 +578,7 @@ abstract class SearchApiAbstractDataSourceController implements SearchApiDataSou } /** - * Set the tracking status of the given items to "changed"/"dirty". - * - * Unless $dequeue is set to TRUE, this operation is ignored for items whose - * status is not "indexed". - * - * @param $item_ids - * Either an array with the IDs of the changed items. Or FALSE to mark all - * items as changed for the given indexes. - * @param array $indexes - * The indexes for which the change should be tracked. - * @param $dequeue - * If set to TRUE, also change the status of queued items. - * - * @throws SearchApiDataSourceException - * If any of the indexes doesn't use the same item type as this controller. + * {@inheritdoc} */ public function trackItemChange($item_ids, array $indexes, $dequeue = FALSE) { if (!$this->table) { @@ -609,21 +602,10 @@ abstract class SearchApiAbstractDataSourceController implements SearchApiDataSou } /** - * Set the tracking status of the given items to "queued". - * - * Queued items are not marked as "dirty" even when they are changed, and they - * are not returned by the getChangedItems() method. - * - * @param $item_ids - * Either an array with the IDs of the queued items. Or FALSE to mark all - * items as queued for the given indexes. - * @param SearchApiIndex $index - * The index for which the items were queued. - * - * @throws SearchApiDataSourceException - * If any of the indexes doesn't use the same item type as this controller. + * {@inheritdoc} */ public function trackItemQueued($item_ids, SearchApiIndex $index) { + $this->checkIndex($index); if (!$this->table) { return; } diff --git a/includes/index_entity.inc b/includes/index_entity.inc index d36b2f8..dc45666 100644 --- a/includes/index_entity.inc +++ b/includes/index_entity.inc @@ -230,7 +230,6 @@ class SearchApiIndex extends Entity { */ public function dequeueItems() { $this->datasource()->stopTracking(array($this)); - _search_api_empty_cron_queue($this); } /** diff --git a/search_api.install b/search_api.install index 8a3366c..f39608c 100644 --- a/search_api.install +++ b/search_api.install @@ -330,7 +330,6 @@ function search_api_disable() { // Modules defining entity or item types might have been disabled. Ignore. } } - DrupalQueue::get('search_api_indexing_queue')->deleteQueue(); } /** @@ -813,3 +812,17 @@ function search_api_update_7114() { } } } + +/** + * Switch to indexing without the use of a cron queue. + */ +function search_api_update_7115() { + variable_del('search_api_batch_per_cron'); + DrupalQueue::get('search_api_indexing_queue')->deleteQueue(); + db_update('search_api_item') + ->fields(array( + 'changed' => 1, + )) + ->condition('changed', 0, '<') + ->execute(); +} diff --git a/search_api.module b/search_api.module index 3faf2d0..597e211 100644 --- a/search_api.module +++ b/search_api.module @@ -255,51 +255,66 @@ function search_api_permission() { * Will index $options['cron-limit'] items for each enabled index. */ function search_api_cron() { - $queue = DrupalQueue::get('search_api_indexing_queue'); - foreach (search_api_index_load_multiple(FALSE, array('enabled' => TRUE, 'read_only' => 0)) as $index) { - $limit = isset($index->options['cron_limit']) + // Load all enabled, not read-only indexes. + $conditions = array( + 'enabled' => TRUE, + 'read_only' => 0 + ); + $indexes = search_api_index_load_multiple(FALSE, $conditions); + if (!$indexes) { + return; + } + // Remember servers which threw an exception. + $ignored_servers = array(); + // Continue indexing, one batch from each index, until the time is up, but at + // least index one batch per index. + $end = time() + variable_get('search_api_index_worker_callback_runtime', 15); + $first_pass = TRUE; + while (TRUE) { + if (!$indexes) { + break; + } + foreach ($indexes as $id => $index) { + if (!$first_pass && time() >= $end) { + break 2; + } + if (!empty($ignored_servers[$index->server])) { + continue; + } + + $limit = isset($index->options['cron_limit']) ? $index->options['cron_limit'] : SEARCH_API_DEFAULT_CRON_LIMIT; - if ($limit) { - try { - $task = array('index' => $index->machine_name); - // Fetch items to index, do not fetch more than the configured amount - // of batches to be created per cron run to avoid timeouts. - $ids = search_api_get_items_to_index($index, $limit > 0 ? $limit * variable_get('search_api_batch_per_cron', 10) : -1); - if (!$ids) { - continue; + $num = 0; + if ($limit) { + try { + $num = search_api_index_items($index, $limit); + if ($num) { + $variables = array( + '@num' => $num, + '%name' => $index->name + ); + watchdog('search_api', 'Indexed @num items for index %name.', $variables, WATCHDOG_INFO); + } } - $batches = $limit > 0 ? array_chunk($ids, $limit, TRUE) : array($ids); - foreach ($batches as $batch) { - $task['items'] = $batch; - $queue->createItem($task); + catch (SearchApiException $e) { + // Exceptions will probably be caused by the server in most cases. + // Therefore, don't index for any index on this server. + $ignored_servers[$index->server] = TRUE; + watchdog_exception('search_api', $e); } - // Mark items as queued so they won't be inserted into the queue again - // on the next cron run. - search_api_track_item_queued($index, $ids); } - catch (SearchApiException $e) { - watchdog_exception('search_api', $e); + if (!$num) { + // Couldn't index any items => stop indexing for this index in this + // cron run. + unset($indexes[$id]); } } + $first_pass = FALSE; } } /** - * Implements hook_cron_queue_info(). - * - * Defines a queue for saved searches that should be checked for new items. - */ -function search_api_cron_queue_info() { - return array( - 'search_api_indexing_queue' => array( - 'worker callback' => '_search_api_indexing_queue_process', - 'time' => variable_get('search_api_index_worker_callback_runtime', 15), - ), - ); -} - -/** * Implements hook_entity_info(). */ function search_api_entity_info() { @@ -686,15 +701,6 @@ function search_api_search_api_index_update(SearchApiIndex $index) { $index->queueItems(); } } - - // If the cron batch size changed, empty the cron queue for this index. - $old_cron = $index->original->options + array('cron_limit' => NULL); - $old_cron = $old_cron['cron_limit']; - $new_cron = $index->options + array('cron_limit' => NULL); - $new_cron = $new_cron['cron_limit']; - if ($old_cron !== $new_cron) { - _search_api_empty_cron_queue($index, TRUE); - } } /** @@ -1132,6 +1138,12 @@ function search_api_track_item_change($type, array $item_ids) { * The index on which items were queued. * @param array $item_ids * The ids of the queued items. + * + * @deprecated + * As of Search API 1.10, the cron queue is not used for indexing anymore, + * therefore this function has become useless. It will, along with + * SearchApiDataSourceControllerInterface::trackItemQueued(), be removed in + * the Drupal 8 version of this module. */ function search_api_track_item_queued(SearchApiIndex $index, array $item_ids) { $index->datasource()->trackItemQueued($item_ids, $index); @@ -1268,53 +1280,31 @@ function _search_api_settings_equals($setting1, $setting2) { } /** - * Indexes items for the specified index. Only items marked as changed are - * indexed, in their order of change (if known). + * Indexes items for the specified index. + * + * Only items marked as changed are indexed, in their order of change (if + * known). * * @param SearchApiIndex $index * The index on which items should be indexed. - * @param $limit - * The number of items which should be indexed at most. -1 means no limit. + * @param int $limit + * (optional) The number of items which should be indexed at most. Defaults to + * -1, which means that all changed items should be indexed. + * + * @return int + * Number of successfully indexed items. * * @throws SearchApiException * If any error occurs during indexing. - * - * @return - * Number of successfully indexed items. */ function search_api_index_items(SearchApiIndex $index, $limit = -1) { - // Don't try to index read-only indexes. + // Don't try to index on read-only indexes. if ($index->read_only) { return 0; } - $queue = DrupalQueue::get('search_api_indexing_queue'); - $queue->createQueue(); - $indexed = 0; - $unlimited = $limit < 0; - $release_items = array(); - while (($unlimited || $indexed < $limit) && ($item = $queue->claimItem(30))) { - if ($item->data['index'] === $index->machine_name) { - $indexed += _search_api_indexing_queue_process($item->data); - $queue->deleteItem($item); - } - else { - $release_items[] = $item; - } - } - - foreach ($release_items as $item) { - $queue->releaseItem($item); - } - - if ($unlimited || $indexed < $limit) { - $ids = search_api_get_items_to_index($index, $unlimited ? -1 : $limit - $indexed); - if ($ids) { - $indexed += count(search_api_index_specific_items($index, $ids)); - } - } - - return $indexed; + $ids = search_api_get_items_to_index($index, $limit); + return $ids ? count(search_api_index_specific_items($index, $ids)) : 0; } /** @@ -1327,11 +1317,11 @@ function search_api_index_items(SearchApiIndex $index, $limit = -1) { * @param array $ids * The IDs of the items which should be indexed. * + * @return array + * The IDs of all successfully indexed items. + * * @throws SearchApiException * If any error occurs during indexing. - * - * @return - * The IDs of all successfully indexed items. */ function search_api_index_specific_items(SearchApiIndex $index, array $ids) { $items = $index->loadItems($ids); @@ -2465,42 +2455,6 @@ function search_api_index_reindex($id) { */ function _search_api_index_reindex(SearchApiIndex $index) { $index->datasource()->trackItemChange(FALSE, array($index), TRUE); - _search_api_empty_cron_queue($index); -} - -/** - * Helper method for removing all of an index's jobs from the cron queue. - * - * @param SearchApiIndex $index - * The index whose jobs should be removed. - * @param $mark_changed - * If TRUE, mark all items in the queue as "changed" again. Defaults to FALSE. - */ -function _search_api_empty_cron_queue(SearchApiIndex $index, $mark_changed = FALSE) { - $index_id = $index->machine_name; - $queue = DrupalQueue::get('search_api_indexing_queue'); - $queue->createQueue(); - $ids = array(); - $release_items = array(); - while ($item = $queue->claimItem()) { - if ($item->data['index'] === $index_id) { - $queue->deleteItem($item); - if ($mark_changed) { - $ids = array_merge($ids, $item->data['items']); - } - } - else { - $release_items[] = $item; - } - } - - foreach ($release_items as $item) { - $queue->releaseItem($item); - } - - if ($ids) { - $index->datasource()->trackItemChange($ids, array($index), TRUE); - } } /** @@ -2553,42 +2507,6 @@ function search_api_index_options_list() { } /** - * Cron queue worker callback for indexing some items. - * - * @param array $task - * An associative array containing: - * - index: The ID of the index on which items should be indexed. - * - items: The items that should be indexed. - * - * @return - * The number of successfully indexed items. - */ -function _search_api_indexing_queue_process(array $task) { - $index = search_api_index_load($task['index']); - try { - if ($index && $index->enabled && !$index->read_only && $task['items']) { - $indexed = search_api_index_specific_items($index, $task['items']); - $num = count($indexed); - // If some items couldn't be indexed, mark them as dirty again. - if ($num < count($task['items'])) { - // Believe it or not but this is actually quite faster than the equivalent - // $diff = array_diff($task['items'], $indexed); - $diff = array_keys(array_diff_key(array_flip($task['items']), array_flip($indexed))); - // Mark the items as dirty again. - $index->datasource()->trackItemChange($diff, array($index), TRUE); - } - if ($num) { - watchdog('search_api', t('Indexed @num items for index @name', array('@num' => $num, '@name' => $index->name)), NULL, WATCHDOG_INFO); - } - return $num; - } - } - catch (SearchApiException $e) { - watchdog_exception('search_api', $e); - } -} - -/** * Shutdown function which indexes all queued items, if any. */ function _search_api_index_queued_items() {