diff --git a/config/install/search_api.settings.yml b/config/install/search_api.settings.yml index 5b9202b..3d5fe5e 100644 --- a/config/install/search_api.settings.yml +++ b/config/install/search_api.settings.yml @@ -1,3 +1,4 @@ default_cron_limit: 50 cron_worker_runtime: 15 default_tracker: 'default' +tracking_page_size: 100 diff --git a/config/schema/search_api.schema.yml b/config/schema/search_api.schema.yml index 7d3ffae..bfbdf22 100644 --- a/config/schema/search_api.schema.yml +++ b/config/schema/search_api.schema.yml @@ -11,3 +11,6 @@ search_api.settings: default_tracker: type: string label: 'The default tracker plugin' + tracking_page_size: + type: integer + label: 'The number of entities loaded at once when adding items to the tracker for a newly created index' diff --git a/search_api.module b/search_api.module index e50774d..800c565 100644 --- a/search_api.module +++ b/search_api.module @@ -5,6 +5,7 @@ * Provides a rich framework for creating searches. */ +use Drupal\Core\Config\ConfigImporter; use Drupal\Core\Entity\ContentEntityInterface; use Drupal\Core\Entity\EntityInterface; use Drupal\search_api\SearchApiException; @@ -37,7 +38,6 @@ function search_api_cron() { // Load all enabled, not read-only indexes. $conditions = array( 'status' => TRUE, - 'read_only' => FALSE, ); /** @var \Drupal\search_api\IndexInterface[] $indexes */ $indexes = entity_load_multiple_by_properties('search_api_index', $conditions); @@ -45,6 +45,21 @@ function search_api_cron() { return; } + // Add items to the tracking system for all indexes for which this hasn't + // happened yet. + $index_task_manager = Utility::getIndexTaskManager(); + foreach ($indexes as $index_id => $index) { + if (!$index_task_manager->isTrackingComplete($index)) { + $index_task_manager->addItemsOnce($index); + } + // Filter out read-only indexes here, since we want to have tracking but not + // index items for them. + if ($index->isReadOnly()) { + unset($indexes[$index_id]); + } + } + + // Now index items. // Remember servers which threw an exception. $ignored_servers = array(); @@ -98,6 +113,22 @@ function search_api_cron() { } /** + * Implements hook_config_import_steps_alter(). + */ +function search_api_config_import_steps_alter(&$sync_steps, ConfigImporter $config_importer) { + $new = $config_importer->getUnprocessedConfiguration('create'); + $changed = $config_importer->getUnprocessedConfiguration('update'); + $new_or_changed = array_merge($new, $changed); + $prefix = \Drupal::entityManager()->getDefinition('search_api_index')->getConfigPrefix() . '.'; + $prefix_length = strlen($prefix); + foreach ($new_or_changed as $config_id) { + if (substr($config_id, 0, $prefix_length) === $prefix) { + $sync_steps[] = array('Drupal\search_api\Utility', 'processIndexTasks'); + } + } +} + +/** * Implements hook_entity_insert(). * * Adds entries for all languages of the new entity to the tracking table for diff --git a/search_api.services.yml b/search_api.services.yml index 36b0cfd..92c73d0 100644 --- a/search_api.services.yml +++ b/search_api.services.yml @@ -19,6 +19,10 @@ services: class: Drupal\search_api\Tracker\TrackerPluginManager parent: default_plugin_manager + search_api.index_task_manager: + class: Drupal\search_api\Task\IndexTaskManager + arguments: ['@state', '@entity.manager'] + search_api.results_static_cache: class: Drupal\search_api\Query\ResultsCache arguments: ['@request_stack'] diff --git a/search_api_db/src/Tests/BackendTest.php b/search_api_db/src/Tests/BackendTest.php index b20e625..a0e87b0 100644 --- a/search_api_db/src/Tests/BackendTest.php +++ b/search_api_db/src/Tests/BackendTest.php @@ -17,6 +17,7 @@ use Drupal\search_api\Entity\Index; use Drupal\search_api\Entity\Server; use Drupal\search_api\Query\ResultSetInterface; use Drupal\search_api\Tests\ExampleContentTrait; +use Drupal\search_api\Utility; use Drupal\system\Tests\Entity\EntityUnitTestBase; use SebastianBergmann\Exporter\Exception; @@ -64,6 +65,8 @@ class BackendTest extends EntityUnitTestBase { $this->setUpExampleStructure(); $this->installConfig(array('search_api_test_db')); + + Utility::getIndexTaskManager()->addItemsAll(Index::load($this->indexId)); } /** diff --git a/src/Datasource/DatasourceInterface.php b/src/Datasource/DatasourceInterface.php index 4f92bbc..14b13cb 100644 --- a/src/Datasource/DatasourceInterface.php +++ b/src/Datasource/DatasourceInterface.php @@ -184,20 +184,21 @@ interface DatasourceInterface extends IndexPluginInterface { /** * Returns a list of IDs of items from this datasource. * - * Returns all items IDs by default. Allows for simple paging by passing - * along a limit and a pointer from where it should start. + * Returns all items IDs by default. However, to avoid issues for large data + * sets, plugins should also implement a paging mechanism (the details of + * which are up to the datasource to decide) which guarantees that all item + * IDs can be retrieved by repeatedly calling this method with increasing + * values for $page (starting with 0) until NULL is returned. * - * @param int $limit - * The amount of items to return - * @param string $from - * The pointer as from where we want to start returning. + * @param int|null $page + * The zero-based page of IDs to retrieve, for the paging mechanism + * implemented by this datasource; or NULL to retrieve all items at once. * - * @return array + * @return string[]|null * An array with datasource-specific (i.e., not prefixed with the datasource - * ID) item IDs. - * - * @todo Change to single $page parameter. + * ID) item IDs; or NULL if there are no more items for this and all + * following pages. */ - public function getItemIds($limit = -1, $from = NULL); + public function getItemIds($page = NULL); } diff --git a/src/Entity/Index.php b/src/Entity/Index.php index 54826fc..21d3baa 100644 --- a/src/Entity/Index.php +++ b/src/Entity/Index.php @@ -1050,29 +1050,6 @@ class Index extends ConfigEntityBase implements IndexInterface { /** * {@inheritdoc} */ - public function startTracking() { - if ($this->hasValidTracker()) { - foreach ($this->getDatasources() as $datasource) { - $item_ids = $datasource->getItemIds(); - if ($item_ids) { - $this->trackItemsInserted($datasource->getPluginId(), $item_ids); - } - } - } - } - - /** - * {@inheritdoc} - */ - public function stopTracking() { - if ($this->hasValidTracker()) { - $this->getTracker()->trackAllItemsDeleted(NULL); - } - } - - /** - * {@inheritdoc} - */ public function trackItemsInserted($datasource_id, array $ids) { $this->trackItemsInsertedOrUpdated($datasource_id, $ids, __FUNCTION__); } @@ -1099,7 +1076,7 @@ class Index extends ConfigEntityBase implements IndexInterface { * "trackItemsUpdated". */ protected function trackItemsInsertedOrUpdated($datasource_id, array $ids, $tracker_method) { - if ($this->hasValidTracker() && $this->status()) { + if ($this->hasValidTracker() && $this->status() && Utility::getIndexTaskManager()->isTrackingComplete($this)) { $item_ids = array(); foreach ($ids as $id) { $item_ids[] = Utility::createCombinedId($datasource_id, $id); @@ -1261,7 +1238,7 @@ class Index extends ConfigEntityBase implements IndexInterface { } elseif (!$this->status() && $original->status()) { if ($this->hasValidTracker()) { - $this->stopTracking(); + Utility::getIndexTaskManager()->stopTracking($this); } if ($original->isServerEnabled()) { $original->getServer()->removeIndex($this); @@ -1270,7 +1247,7 @@ class Index extends ConfigEntityBase implements IndexInterface { elseif ($this->status() && !$original->status()) { $this->getServer()->addIndex($this); if ($this->hasValidTracker()) { - $this->startTracking(); + Utility::getIndexTaskManager()->startTracking($this); } } @@ -1324,16 +1301,11 @@ class Index extends ConfigEntityBase implements IndexInterface { $new_datasource_ids = $this->getDatasourceIds(); $original_datasource_ids = $original->getDatasourceIds(); if ($new_datasource_ids != $original_datasource_ids) { - $removed = array_diff($original_datasource_ids, $new_datasource_ids); $added = array_diff($new_datasource_ids, $original_datasource_ids); - foreach ($removed as $datasource_id) { - $this->getTracker()->trackAllItemsDeleted($datasource_id); - } - foreach ($added as $datasource_id) { - $datasource = $this->getDatasource($datasource_id); - $item_ids = $datasource->getItemIds(); - $this->trackItemsInserted($datasource_id, $item_ids); - } + $removed = array_diff($original_datasource_ids, $new_datasource_ids); + $index_task_manager = Utility::getIndexTaskManager(); + $index_task_manager->stopTracking($this, $removed); + $index_task_manager->startTracking($this, $added); } } @@ -1349,11 +1321,12 @@ class Index extends ConfigEntityBase implements IndexInterface { */ protected function reactToTrackerSwitch(IndexInterface $original) { if ($this->tracker != $original->getTrackerId()) { + $index_task_manager = Utility::getIndexTaskManager(); if ($original->hasValidTracker()) { - $original->stopTracking(); + $index_task_manager->stopTracking($this); } if ($this->hasValidTracker()) { - $this->startTracking(); + $index_task_manager->startTracking($this); } } } diff --git a/src/Form/IndexForm.php b/src/Form/IndexForm.php index 8db50d3..447f0df 100644 --- a/src/Form/IndexForm.php +++ b/src/Form/IndexForm.php @@ -8,7 +8,6 @@ namespace Drupal\search_api\Form; use Drupal\Component\Utility\Html; -use Drupal\Component\Utility\SafeMarkup; use Drupal\Core\Cache\Cache; use Drupal\Core\Entity\EntityForm; use Drupal\Core\Entity\EntityManagerInterface; @@ -17,6 +16,7 @@ use Drupal\search_api\Datasource\DatasourcePluginManager; use Drupal\search_api\IndexInterface; use Drupal\search_api\SearchApiException; use Drupal\search_api\Tracker\TrackerPluginManager; +use Drupal\search_api\Utility; use Symfony\Component\DependencyInjection\ContainerInterface; /** @@ -572,9 +572,15 @@ class IndexForm extends EntityForm { // Only save the index if the form doesn't need to be rebuilt. if (!$form_state->isRebuilding()) { try { - $this->getEntity()->save(); + /** @var \Drupal\search_api\IndexInterface $index */ + $index = $this->getEntity(); + $index->save(); drupal_set_message($this->t('The index was successfully saved.')); - $form_state->setRedirect('entity.search_api_index.canonical', array('search_api_index' => $this->getEntity()->id())); + $form_state->setRedirect('entity.search_api_index.canonical', array('search_api_index' => $index->id())); + $index_task_manager = Utility::getIndexTaskManager(); + if (!$index_task_manager->isTrackingComplete($index)) { + $index_task_manager->addItemsBatch($index); + } } catch (SearchApiException $ex) { $form_state->setRebuild(); diff --git a/src/Form/IndexStatusForm.php b/src/Form/IndexStatusForm.php index e5bdf17..1d5e6f6 100644 --- a/src/Form/IndexStatusForm.php +++ b/src/Form/IndexStatusForm.php @@ -13,6 +13,7 @@ use Drupal\Core\Form\FormStateInterface; use Drupal\search_api\IndexBatchHelper; use Drupal\search_api\SearchApiException; use Drupal\search_api\IndexInterface; +use Drupal\search_api\Utility; /** * Provides a form for indexing, clearing, etc., an index. @@ -30,11 +31,32 @@ class IndexStatusForm extends FormBase { * {@inheritdoc} */ public function buildForm(array $form, FormStateInterface $form_state, IndexInterface $index = NULL) { + if (!isset($index)) { + return array(); + } + $form['#index'] = $index; $form['#attached']['library'][] = 'search_api/drupal.search_api.admin_css'; if ($index->hasValidTracker()) { + if (!Utility::getIndexTaskManager()->isTrackingComplete($index)) { + $form['tracking'] = array( + '#type' => 'details', + '#title' => $this->t('Track items for index'), + '#description' => $this->t('Not all items have been tracked for this index. This means the displayed index status is incomplete and not all items will currently be indexed.'), + '#open' => TRUE, + '#attributes' => array( + 'class' => array('container-inline'), + ), + ); + $form['tracking']['index_now'] = array( + '#type' => 'submit', + '#value' => $this->t('Track now'), + '#name' => 'track_now', + ); + } + // Add the "Index now" form. $form['index'] = array( '#type' => 'details', @@ -178,6 +200,10 @@ class IndexStatusForm extends FormBase { case 'clear': $form_state->setRedirect('entity.search_api_index.clear', array('search_api_index' => $index->id())); break; + + case 'track_now': + Utility::getIndexTaskManager()->addItemsBatch($index); + break; } } diff --git a/src/IndexInterface.php b/src/IndexInterface.php index db69495..6f7a26c 100644 --- a/src/IndexInterface.php +++ b/src/IndexInterface.php @@ -462,22 +462,6 @@ interface IndexInterface extends ConfigEntityInterface { public function indexItems(array $search_objects); /** - * Starts tracking for this index. - * - * This is normally done when the index is first created (if it is enabled) or - * when it is enabled. - */ - public function startTracking(); - - /** - * Stops tracking for this index. - * - * This is done when the index is disabled, or when an enabled index is - * deleted. - */ - public function stopTracking(); - - /** * Adds items from a specific datasource to the index. * * Note that this method receives datasource-specific item IDs as the diff --git a/src/Plugin/search_api/datasource/ContentEntity.php b/src/Plugin/search_api/datasource/ContentEntity.php index caf97c3..0f35813 100644 --- a/src/Plugin/search_api/datasource/ContentEntity.php +++ b/src/Plugin/search_api/datasource/ContentEntity.php @@ -9,6 +9,7 @@ namespace Drupal\search_api\Plugin\search_api\datasource; use Drupal\Component\Utility\Html; use Drupal\Component\Utility\SafeMarkup; +use Drupal\Core\Config\ConfigFactoryInterface; use Drupal\Core\Entity\ContentEntityInterface; use Drupal\Core\Entity\EntityInterface; use Drupal\Core\Entity\EntityManagerInterface; @@ -49,6 +50,13 @@ class ContentEntity extends DatasourcePluginBase { protected $typedDataManager; /** + * The config factory. + * + * @var \Drupal\Core\Config\ConfigFactoryInterface|null + */ + protected $configFactory; + + /** * {@inheritdoc} */ public function __construct(array $configuration, $plugin_id, array $plugin_definition) { @@ -80,6 +88,10 @@ class ContentEntity extends DatasourcePluginBase { $typed_data_manager = $container->get('typed_data_manager'); $datasource->setTypedDataManager($typed_data_manager); + /** @var $config_factory \Drupal\Core\Config\ConfigFactoryInterface */ + $config_factory = $container->get('config.factory'); + $datasource->setConfigFactory($config_factory); + return $datasource; } @@ -150,6 +162,41 @@ class ContentEntity extends DatasourcePluginBase { } /** + * Retrieves the config factory. + * + * @return \Drupal\Core\Config\ConfigFactoryInterface + * The config factory. + */ + public function getConfigFactory() { + return $this->configFactory ?: \Drupal::configFactory(); + } + + /** + * Retrieves the config value for a certain key in the Search API settings. + * + * @param string $key + * The key whose value should be retrieved. + * + * @return mixed + * The config value for the given key. + */ + protected function getConfigValue($key) { + return $this->getConfigFactory()->get('search_api.settings')->get($key); + } + + /** + * Sets the config factory. + * + * @param \Drupal\Core\Config\ConfigFactoryInterface $config_factory + * The new config factory. + * + * @return $this + */ + public function setConfigFactory(ConfigFactoryInterface $config_factory) { + $this->configFactory = $config_factory; + return $this; + } + /** * {@inheritdoc} */ public function getPropertyDefinitions() { @@ -380,9 +427,8 @@ class ContentEntity extends DatasourcePluginBase { /** * {@inheritdoc} */ - public function getItemIds($limit = '-1', $from = NULL) { - // @todo Implement paging. - return $this->getBundleItemIds(); + public function getItemIds($page = NULL) { + return $this->getBundleItemIds(NULL, $page); } /** @@ -436,11 +482,14 @@ class ContentEntity extends DatasourcePluginBase { * @param string[]|null $bundles * (optional) The bundles for which all item IDs should be returned; or NULL * to retrieve IDs from all enabled bundles in this datasource. + * @param int|null $page + * The zero-based page of IDs to retrieve, for the paging mechanism + * implemented by this datasource; or NULL to retrieve all items at once. * * @return string[] * An array of all item IDs of these bundles. */ - protected function getBundleItemIds(array $bundles = NULL) { + protected function getBundleItemIds(array $bundles = NULL, $page = NULL) { // If NULL was passed, use all enabled bundles. if (!isset($bundles)) { $bundles = array_keys($this->getBundles()); @@ -454,8 +503,16 @@ class ContentEntity extends DatasourcePluginBase { $select->condition($this->getEntityType()->getKey('bundle'), $bundles, 'IN'); } } + if (isset($page)) { + $page_size = $this->getConfigValue('tracking_page_size'); + $select->range($page * $page_size, $page_size); + } $entity_ids = $select->execute(); + if (!$entity_ids) { + return NULL; + } + // For all the loaded entities, compute all their item IDs (one for each // translation). $item_ids = array(); diff --git a/src/Task/IndexTaskManager.php b/src/Task/IndexTaskManager.php new file mode 100644 index 0000000..cbb287a --- /dev/null +++ b/src/Task/IndexTaskManager.php @@ -0,0 +1,292 @@ +state = $state; + $this->entityManager = $entity_manager; + } + + /** + * {@inheritdoc} + */ + public function startTracking(IndexInterface $index, array $datasource_ids = NULL) { + $index_state = $this->getIndexState($index); + if (empty($index_state['status'])) { + if (!isset($datasource_ids)) { + $datasource_ids = $index->getDatasourceIds(); + } + elseif (!$datasource_ids) { + // Calling with an empty array as $datasource_ids could otherwise break + // this, so we have to check for this situation. + return; + } + $index_state = array( + 'status' => 1, + 'pages' => array_fill_keys($datasource_ids, 0), + ); + $this->state->set($this->getIndexStateKey($index), $index_state); + } + } + + /** + * {@inheritdoc} + */ + public function addItemsOnce(IndexInterface $index) { + $index_state = $this->getIndexState($index); + if (!($index_state['status'] && $index_state['pages'])) { + return NULL; + } + + if (!$index->hasValidTracker()) { + return 0; + } + + // Use this method to automatically circle through the datasources, adding + // items from each of them in turn. + $page = reset($index_state['pages']); + $datasource_id = key($index_state['pages']); + unset($index_state['pages'][$datasource_id]); + + $added = 0; + if ($index->isValidDatasource($datasource_id)) { + $raw_ids = $index->getDatasource($datasource_id)->getItemIds($page); + if ($raw_ids !== NULL) { + $index_state['pages'][$datasource_id] = ++$page; + if ($raw_ids) { + $item_ids = array(); + foreach ($raw_ids as $raw_id) { + $item_ids[] = Utility::createCombinedId($datasource_id, $raw_id); + } + $added = count($item_ids); + $index->getTracker()->trackItemsInserted($item_ids); + } + } + } + + if (empty($index_state['pages'])) { + $this->state->delete($this->getIndexStateKey($index)); + return NULL; + } + + $this->state->set($this->getIndexStateKey($index), $index_state); + return $added; + } + + /** + * {@inheritdoc} + */ + public function addItemsBatch(IndexInterface $index) { + $index_state = $this->getIndexState($index); + if (!empty($index_state['status'])) { + $batch_definition = array( + 'operations' => array( + array(array($this, 'processBatch'), array($index->id())), + ), + 'finished' => array($this, 'finishBatch'), + ); + // Schedule the batch. + batch_set($batch_definition); + } + } + + /** + * Adds one page of items to the tracker as part of a batch operation. + * + * @param string $index_id + * The ID of the index on which items should be indexed. + * @param array $context + * The current batch context, as defined in the @link batch Batch operations + * @endlink documentation. + */ + public function processBatch($index_id, array &$context) { + /** @var \Drupal\search_api\IndexInterface $index */ + $index = $this->entityManager->getStorage('search_api_index')->load($index_id); + $added = $this->addItemsOnce($index); + if (!isset($context['results']['added'])) { + $context['results']['added'] = 0; + } + if ($added === NULL) { + $context['finished'] = 1; + } + else { + $context['finished'] = 0; + $context['results']['added'] += $added; + $args['%index'] = $index->label(); + $context['message'] = $this->formatPlural($context['results']['added'], 'Tracked 1 item for the %index index.', 'Tracked @count items for the %index index.', $args); + } + } + + /** + * Finishes a "start tracking" batch. + * + * @param bool $success + * Indicates whether the batch process was successful. + * @param array $results + * Results information passed from the processing callback. + */ + public function finishBatch($success, $results) { + // Check if the batch job was successful. + if ($success) { + // Display the number of items tracked. + $indexed_message = $this->formatPlural($results['added'], 'Successfully tracked 1 item for this index.', 'Successfully tracked @count items for this index.'); + drupal_set_message($indexed_message); + } + else { + // Notify the user about the batch job failure. + drupal_set_message($this->t('An error occurred while trying to start tracking. Check the logs for details.'), 'error'); + } + } + + /** + * {@inheritdoc} + */ + public function addItemsAll(IndexInterface $index) { + do { + $return = $this->addItemsOnce($index); + } while ($return !== NULL); + } + + /** + * {@inheritdoc} + */ + public function stopTracking(IndexInterface $index, array $datasource_ids = NULL) { + $valid_tracker = $index->hasValidTracker(); + if (!isset($datasource_ids)) { + $this->state->delete($this->getIndexStateKey($index)); + if ($valid_tracker) { + $index->getTracker()->trackAllItemsDeleted(); + } + return; + } + + // Catch the case of being called with an empty array of datasources. + if (!$datasource_ids) { + return; + } + + // If no state is saved, this will return NULL, making the following unset() + // statements no-ops. + $index_state = $this->getIndexState($index, FALSE); + foreach ($datasource_ids as $datasource_id) { + unset($index_state['pages'][$datasource_id]); + if ($valid_tracker) { + $index->getTracker()->trackAllItemsDeleted($datasource_id); + } + } + + // If we had an index state saved, update it now. + if (isset($index_state)) { + if (empty($index_state['pages'])) { + $this->state->delete($this->getIndexStateKey($index)); + } + else { + $this->state->set($this->getIndexStateKey($index), $index_state); + } + } + } + + /** + * {@inheritdoc} + */ + public function isTrackingComplete(IndexInterface $index) { + $index_state = $this->getIndexState($index); + return !$index_state['status']; + } + + /** + * Retrieves the current state for a specific index. + * + * @param \Drupal\search_api\IndexInterface $index + * The index whose state should be retrieved. + * @param bool $provide_default + * Whether to provide a default state array if there was no state saved. + * + * @return array|null + * An associative array with information about the state of the given index: + * - status: 0 if no tracking operation is currently in progress, 1 if there + * is. + * - page: If status is 1, the next page of items to process for tracking. + * Or NULL if there was no saved state and $provide_default is FALSE. + */ + protected function getIndexState(IndexInterface $index, $provide_default = TRUE) { + $default = array( + 'status' => 0, + 'pages' => array(), + ); + return $this->state->get($this->getIndexStateKey($index), $provide_default ? $default : NULL); + } + + /** + * Retrieves the key of the index's state. + * + * @param \Drupal\search_api\IndexInterface $index + * The index. + * + * @return string + * The key which identifies the index's state in the site state. + */ + protected function getIndexStateKey(IndexInterface $index) { + return 'search_api.index.' . $index->id() . '.tasks'; + } + + /** + * Implements the magic __sleep() method. + * + * Prevents the any properties from being serialized. + */ + public function __sleep() { + return array(); + } + + /** + * Implements the magic __wakeup() method. + * + * Reloads the services into this object's properties. + */ + public function __wakeup() { + $this->state = \Drupal::state(); + $this->entityManager = \Drupal::entityManager(); + } + +} diff --git a/src/Task/IndexTaskManagerInterface.php b/src/Task/IndexTaskManagerInterface.php new file mode 100644 index 0000000..5754bd8 --- /dev/null +++ b/src/Task/IndexTaskManagerInterface.php @@ -0,0 +1,87 @@ + array('index_directly' => FALSE), )); $this->index->save(); + + Utility::getIndexTaskManager()->addItemsAll($this->index); } /** diff --git a/src/Tests/Processor/ContentAccessTest.php b/src/Tests/Processor/ContentAccessTest.php index e2fc710..75fd3da 100644 --- a/src/Tests/Processor/ContentAccessTest.php +++ b/src/Tests/Processor/ContentAccessTest.php @@ -112,6 +112,7 @@ class ContentAccessTest extends ProcessorTestBase { $this->index->set('datasources', array('entity:comment', 'entity:node', 'entity:user')); $this->index->save(); + Utility::getIndexTaskManager()->addItemsAll($this->index); $this->index = entity_load('search_api_index', $this->index->id(), TRUE); } diff --git a/src/Tests/Processor/ProcessorTestBase.php b/src/Tests/Processor/ProcessorTestBase.php index 940997f..29e4c89 100644 --- a/src/Tests/Processor/ProcessorTestBase.php +++ b/src/Tests/Processor/ProcessorTestBase.php @@ -106,6 +106,11 @@ abstract class ProcessorTestBase extends EntityUnitTestBase { $this->processor = $plugin_manager->createInstance($processor, array('index' => $this->index)); } $this->index->save(); + \Drupal::configFactory() + ->getEditable('search_api.settings') + ->set('tracking_page_size', 100) + ->save(); + Utility::getIndexTaskManager()->addItemsAll($this->index); } /** diff --git a/src/Tests/ViewsTest.php b/src/Tests/ViewsTest.php index c0f45a3..04e6960 100644 --- a/src/Tests/ViewsTest.php +++ b/src/Tests/ViewsTest.php @@ -7,6 +7,9 @@ namespace Drupal\search_api\Tests; +use Drupal\search_api\Entity\Index; +use Drupal\search_api\Utility; + /** * Tests the Views integration of the Search API. * @@ -37,6 +40,8 @@ class ViewsTest extends WebTestBase { parent::setUp(); $this->setUpExampleStructure(); + + Utility::getIndexTaskManager()->addItemsAll(Index::load($this->indexId)); } /** diff --git a/src/Utility.php b/src/Utility.php index 0641e6c..73a03c8 100644 --- a/src/Utility.php +++ b/src/Utility.php @@ -7,6 +7,7 @@ namespace Drupal\search_api; +use Drupal\Core\Config\ConfigImporter; use Drupal\Core\Entity\EntityInterface; use Drupal\Core\TypedData\ComplexDataDefinitionInterface; use Drupal\Core\TypedData\ComplexDataInterface; @@ -14,6 +15,7 @@ use Drupal\Core\TypedData\DataReferenceInterface; use Drupal\Core\TypedData\ListInterface; use Drupal\Core\TypedData\TypedDataInterface; use Drupal\search_api\Datasource\DatasourceInterface; +use Drupal\search_api\Entity\Index; use Drupal\search_api\Item\AdditionalField; use Drupal\search_api\Item\Field; use Drupal\search_api\Item\FieldInterface; @@ -311,6 +313,72 @@ class Utility { } /** + * Retrieves the index task manager. + * + * @return \Drupal\search_api\Task\IndexTaskManagerInterface + * The index task manager. + */ + public static function getIndexTaskManager() { + return \Drupal::service('search_api.index_task_manager'); + } + + /** + * Processes all pending index tasks inside a batch run. + * + * @param array $context + * The current batch context. + * @param \Drupal\Core\Config\ConfigImporter $config_importer + * The config importer. + */ + public static function processIndexTasks(array &$context, ConfigImporter $config_importer) { + $index_task_manager = static::getIndexTaskManager(); + + if (!isset($context['sandbox']['indexes'])) { + $context['sandbox']['indexes'] = array(); + + $indexes = \Drupal::entityManager() + ->getStorage('search_api_index') + ->loadByProperties(array( + 'status' => TRUE, + )); + $deleted = $config_importer->getUnprocessedConfiguration('delete'); + + /** @var \Drupal\search_api\IndexInterface $index */ + foreach ($indexes as $index_id => $index) { + if (!$index_task_manager->isTrackingComplete($index) && !in_array($index->getConfigDependencyName(), $deleted)) { + $context['sandbox']['indexes'][] = $index_id; + } + } + $context['sandbox']['total'] = count($context['sandbox']['indexes']); + if (!$context['sandbox']['total']) { + $context['finished'] = 1; + return; + } + } + + $index_id = array_shift($context['sandbox']['indexes']); + $index = Index::load($index_id); + $added = $index_task_manager->addItemsOnce($index); + if ($added !== NULL) { + array_unshift($context['sandbox']['indexes'], $index_id); + } + + if (empty($context['sandbox']['indexes'])) { + $context['finished'] = 1; + } + else { + $finished = $context['sandbox']['total'] - count($context['sandbox']['indexes']); + $context['finished'] = $finished / $context['sandbox']['total']; + $args = array( + '%index' => $index->label(), + '@num' => $finished + 1, + '@total' => $context['sandbox']['total'], + ); + $context['message'] = \Drupal::translation()->translate('Tracking items for search index %index (@num of @total)', $args); + } + } + + /** * Creates a new search query object. * * @param \Drupal\search_api\IndexInterface $index