diff --git a/config/schema/search_api.schema.yml b/config/schema/search_api.schema.yml index 6e6bcd0..0725eb6 100644 --- a/config/schema/search_api.schema.yml +++ b/config/schema/search_api.schema.yml @@ -62,6 +62,11 @@ search_api.index.*: label: 'Datasource plugin ID' datasourcePluginConfig: type: search_api.datasource.plugin.[%parent.datasourcePluginId] + trackerPluginId: + type: string + label: 'Tracker plugin ID' + trackerPluginConfig: + type: search_api.tracker.plugin.[%parent.trackerPluginId] serverMachineName: type: string label: 'Server machine name' diff --git a/config/search_api.index.default_node_index.yml b/config/search_api.index.default_node_index.yml index e42ca04..c789ce8 100644 --- a/config/search_api.index.default_node_index.yml +++ b/config/search_api.index.default_node_index.yml @@ -55,5 +55,7 @@ datasourcePluginId: 'entity:node' datasourcePluginConfig: default: '1' bundles: { } +trackerPluginId: 'default_tracker' +trackerPluginConfig: { } serverMachineName: '' status: true diff --git a/config/search_api.settings.yml b/config/search_api.settings.yml index bd19987..5afe40d 100644 --- a/config/search_api.settings.yml +++ b/config/search_api.settings.yml @@ -1,2 +1,3 @@ cron_worker_runtime: '15' cron_limit: 50 +default_tracker: 'default_tracker' diff --git a/lib/Drupal/search_api/Annotation/SearchApiTracker.php b/lib/Drupal/search_api/Annotation/SearchApiTracker.php new file mode 100644 index 0000000..667e4fb --- /dev/null +++ b/lib/Drupal/search_api/Annotation/SearchApiTracker.php @@ -0,0 +1,44 @@ +hasValidTracker() ? $index->getTracker()->getRemainingItemsCount() : 0); // Check if an explicit limit needs to be used. if ($context['sandbox']['limit'] > -1) { - // Calculate the remaining amount of items that can be indexed. - $actual_limit = $context['sandbox']['limit'] - $context['sandbox']['progress']; + // Calculate the remaining amount of items that can be indexed. Note that + // a minimum is taking between the allowed number of items and the + // remaining item count to prevent incorrect reporting of not indexed + // items. + $actual_limit = min($context['sandbox']['limit'] - $context['sandbox']['progress'], $remaining_item_count); } else { - // Use the remaining item count as actual limit. When no valid datasource - // is available then the actual limit will be set to zero and result - // in no items being index. This will in turn cause the batch job to - // stop. - $actual_limit = ($index->hasValidDatasource() ? $index->getDatasource()->getRemainingItemsCount() : 0); + // Use the remaining item count as actual limit. + $actual_limit = $remaining_item_count; } // Determine the number of items to index for this run. $to_index = min($actual_limit, $context['sandbox']['batch_size']); @@ -125,7 +129,7 @@ class IndexBatchHelper { $indexed = $index->index($to_index); // Increment the indexed result and progress. $context['results']['indexed'] += $indexed; - $context['results']['not indexed'] += ($actual_limit - $indexed); + $context['results']['not indexed'] += ($to_index - $indexed); $context['sandbox']['progress'] += $to_index; // Display progress message. if ($indexed > 0) { diff --git a/lib/Drupal/search_api/Datasource/DatasourceInterface.php b/lib/Drupal/search_api/Datasource/DatasourceInterface.php index a32b911..92e23ca 100644 --- a/lib/Drupal/search_api/Datasource/DatasourceInterface.php +++ b/lib/Drupal/search_api/Datasource/DatasourceInterface.php @@ -69,95 +69,6 @@ interface DatasourceInterface extends IndexPluginInterface { public function stopTracking(); /** - * Track IDs as inserted. - * - * @param array $ids - * An array of item IDs. - * - * @return boolean - * TRUE if successful, otherwise FALSE. - */ - public function trackInsert(array $ids); - - /** - * Track IDs as updated. - * - * @param array $ids - * An array of item IDs, or NULL to mark all items as changed. - * - * @return boolean - * TRUE if successful, otherwise FALSE. - */ - public function trackUpdate(array $ids = NULL); - - /** - * Track IDs as indexed. - * - * @param array $ids - * An array of item IDs. - * - * @return boolean - * TRUE if successful, otherwise FALSE. - */ - public function trackIndexed(array $ids); - - /** - * Track IDs as deleted. - * - * @param array|NULL $ids - * An array of item IDs. - */ - public function trackDelete(array $ids = NULL); - - /** - * Clear all tracked items. - * - * @return boolean - * TRUE if successful, otherwise FALSE. - */ - public function clear(); - - /** - * Get a list of IDs that need to be indexed. - * - * If possible, completely unindexed items should be returned before items - * that were indexed but later changed. Also, items that were changed longer - * ago should be favored. - * - * @param integer $limit - * Optional. The maximum number of items to return. Negative values mean - * "unlimited". Defaults to all changed items. - * - * @return array - * An array of item IDs that need to be indexed for the given index. - */ - public function getRemainingItems($limit = -1); - - /** - * Retrieves the number of indexed items. - * - * @return int - * The number of indexed items. - */ - public function getIndexedItemsCount(); - - /** - * Retrieves the number of changed items. - * - * @return int - * The number of changed items. - */ - public function getRemainingItemsCount(); - - /** - * Retrieves the total number of items that have to be indexed. - * - * @return int - * The total number of items that have to be indexed. - */ - public function getTotalItemsCount(); - - /** * Returns view mode info for this item type. * * @return array diff --git a/lib/Drupal/search_api/Datasource/DatasourcePluginBase.php b/lib/Drupal/search_api/Datasource/DatasourcePluginBase.php index 1baa80b..633cdc1 100644 --- a/lib/Drupal/search_api/Datasource/DatasourcePluginBase.php +++ b/lib/Drupal/search_api/Datasource/DatasourcePluginBase.php @@ -6,10 +6,8 @@ namespace Drupal\search_api\Datasource; -use Drupal\Core\Database\Connection; use Drupal\Core\TypedData\ComplexDataInterface; use Drupal\search_api\Plugin\IndexPluginBase; -use Symfony\Component\DependencyInjection\ContainerInterface; /** * Defines a base class from which other datasources may extend. @@ -35,54 +33,6 @@ use Symfony\Component\DependencyInjection\ContainerInterface; abstract class DatasourcePluginBase extends IndexPluginBase implements DatasourceInterface { /** - * The database connection. - * - * @var \Drupal\Core\Database\Connection - */ - protected $databaseConnection; - - - /** - * The table to use for tracking. - * - * @var string - */ - protected $table; - - /** - * Create a DatasourcePluginBase object. - * - * @param \Drupal\Core\Database\Connection $connection - * A connection to the database. - * @param string $table - * The table to use for tracking. - * @param array $configuration - * A configuration array containing information about the plugin instance. - * @param string $plugin_id - * The plugin_id for the plugin instance. - * @param array $plugin_definition - * The plugin implementation definition. - */ - public function __construct(Connection $connection, array $configuration, $plugin_id, array $plugin_definition, $table = 'search_api_item') { - // Initialize the parent chain of objects. - parent::__construct($configuration, $plugin_id, $plugin_definition); - // Setup object members. - $this->databaseConnection = $connection; - $this->table = $table; - } - - /** - * {@inheritdoc} - */ - public static function create(ContainerInterface $container, array $configuration, $plugin_id, $plugin_definition) { - /** @var $connection \Drupal\Core\Database\Connection */ - $connection = $container->get('database'); - // @todo: Make this more dynamic - $table = 'search_api_item'; - return new static($connection, $table, $configuration, $plugin_id, $plugin_definition); - } - - /** * Retrieves the index. * * @return \Drupal\search_api\Index\IndexInterface @@ -93,332 +43,6 @@ abstract class DatasourcePluginBase extends IndexPluginBase implements Datasourc } /** - * Retrieves the database connection - * - * @return \Drupal\Core\Database\Connection - * An instance of Connection. - */ - protected function getDatabaseConnection() { - return $this->databaseConnection; - } - - /** - * Creates a delete statement. - * - * @return \Drupal\Core\Database\Query\Delete - * An instance of ConditionInterface. - */ - protected function createDeleteStatement() { - return $this->getDatabaseConnection()->delete('search_api_item') - ->condition('index_id', $this->getIndex()->id()); - } - - /** - * Creates a select statement. - * - * @return \Drupal\Core\Database\Query\SelectInterface - * An instance of SelectInterface. - */ - protected function createSelectStatement() { - return $this->getDatabaseConnection()->select('search_api_item', 'sai') - ->condition('index_id', $this->getIndex()->id()); - } - - /** - * Creates an insert statement. - * - * @return \Drupal\Core\Database\Query\Insert - * An instance of Insert. - */ - protected function createInsertStatement() { - return $this->getDatabaseConnection()->insert('search_api_item') - ->fields(array('item_id', 'index_id', 'changed')); - } - - /** - * Creates an update statement. - * - * @return \Drupal\Core\Database\Query\Update - * An instance of Update. - */ - protected function createUpdateStatement() { - return $this->getDatabaseConnection()->update('search_api_item') - ->condition('index_id', $this->getIndex()->id()); - } - - /** - * {@inheritdoc} - */ - public function trackInsert(array $ids) { - // Initialize the success variable to FALSE. - $success = FALSE; - // Get the index. - $index = $this->getIndex(); - // Check if the index is enabled, writable and exists. - if ($index->status() && !$index->isReadOnly()) { - // Start a database transaction. - $transaction = $this->getDatabaseConnection()->startTransaction(); - // Get the index ID. - $index_id = $index->id(); - // Catch any exception that may occur during insert. - try { - // Iterate through the IDs in chunks of 1000 items. - foreach (array_chunk($ids, 1000) as $ids_chunk) { - // Build the insert statement. - $statement = $this->createInsertStatement(); - // Iterate through the chunked IDs. - foreach ($ids_chunk as $item_id) { - // Add the ID to the insert statement. - $statement->values(array( - 'item_id' => $item_id, - 'index_id' => $index_id, - 'changed' => 0, - )); - } - // Execute the insert statement. - $statement->execute(); - } - // Indicate successful operation. - $success = TRUE; - } - catch (\Exception $ex) { - // Log the exception to watchdog. - watchdog_exception('Search API', $ex); - // Rollback the transaction. - $transaction->rollback(); - } - } - return $success; - } - - /** - * {@inheritdoc} - */ - public function trackUpdate(array $ids = NULL) { - // Initialize the success variable to FALSE. - $success = FALSE; - // Get the index. - $index = $this->getIndex(); - // Check if the index is enabled, writable and exists. - if ($index->status() && !$index->isReadOnly()) { - // Get the database connection. - $connection = $this->getDatabaseConnection(); - // Start a database transaction. - $transaction = $connection->startTransaction(); - // Catch any exception that may occur during update. - try { - // Iterate through the IDs in chunks of 1000 items. - $ids_chunks = $ids ? array_chunk($ids, 1000) : array(NULL); - foreach ($ids_chunks as $ids_chunk) { - // Build the fields which need to be updated. - // Build the update statement. - $statement = $this->createUpdateStatement() - ->fields(array( - 'changed' => REQUEST_TIME, - )); - if ($ids_chunk) { - $statement->condition('item_id', $ids_chunk); - } - // Execute the update statement. - $statement->execute(); - } - // Indicate success operation. - $success = TRUE; - } - catch (\Exception $ex) { - // Log the exception to watchdog. - watchdog_exception('Search API', $ex); - // Rollback the transaction. - $transaction->rollback(); - } - } - return $success; - } - - /** - * {@inheritdoc} - */ - public function trackIndexed(array $ids) { - // Initialize the success variable to FALSE. - $success = FALSE; - // Get the index. - $index = $this->getIndex(); - // Check if the index is enabled, writable and exists. - if ($index->status() && !$index->isReadOnly() && !$index->isNew()) { - // Get the database connection. - $connection = $this->getDatabaseConnection(); - // Start a database transaction. - $transaction = $connection->startTransaction(); - // Catch any exception that may occur during update. - try { - // Iterate through the IDs in chunks of 1000 items. - foreach (array_chunk($ids, 1000) as $ids_chunk) { - // Build the fields which need to be updated. - $fields = array( - 'changed' => REQUEST_TIME, - ); - // Build and execute the update statement. - $this->createUpdateStatement() - ->fields($fields) - ->condition('item_id', $ids_chunk) - ->execute(); - } - // Indicate success operation. - $success = TRUE; - } - catch (\Exception $ex) { - // Log the exception to watchdog. - watchdog_exception('Search API', $ex); - // Rollback the transaction. - $transaction->rollback(); - } - } - return $success; - } - - /** - * {@inheritdoc} - */ - public function trackDelete(array $ids = NULL) { - $delete_statement = $this->createDeleteStatement(); - if (isset($ids)) { - $delete_statement->condition('item_id', $ids, 'IN'); - } - $delete_statement->execute(); - } - - /** - * {@inheritdoc} - */ - public function clear() { - // Initialize the success variable to FALSE. - $success = FALSE; - // Get the index. - $index = $this->getIndex(); - // Check if the index is enabled, writable and exists. - if ($index->status() && !$index->isReadOnly() && !$index->isNew()) { - // Get the database connection. - $connection = $this->getDatabaseConnection(); - // Start a database transaction. - $transaction = $connection->startTransaction(); - // Catch any exception that may occur during update. - try { - // Build and execute the update statement. - $this->trackDelete(); - // Indicate success operation. - $success = TRUE; - } - catch (\Exception $ex) { - // Log the exception to watchdog. - watchdog_exception('Search API', $ex); - // Rollback the transaction. - $transaction->rollback(); - } - } - return $success; - } - - /** - * Creates the base query so we can get al the items or just the count - * - * @return \Drupal\Core\Database\Query\SelectInterface - */ - private function getRemainingItemsQuery() { - // Get the index and its last state - $index = $this->getIndex(); - - // Get $last_entity_id and $last_changed. - $last_indexed = $index->getLastIndexed(); - - // Build the select statement. - $statement = $this->createSelectStatement(); - - - $changed = $last_indexed['changed']; - $item_id = $last_indexed['item_id']; - - // Find the next batch of entities to index for this entity type. Note that - // for ordering we're grabbing the oldest first and then ordering by ID so - // that we get a definitive order. - // Also note that we fetch ALL fields from the indexer table - $alias = $statement->addExpression('LENGTH(item_id)', 'item_id_length'); - - $statement - ->fields('sai', array('item_id')) - ->condition(db_or() - ->condition('sai.changed', $changed, '>') - // Tie breaker for entities that were changed at exactly - // the same second as the last indexed entity - ->condition(db_and() - ->condition('sai.changed', $changed, '=') - ->condition('sai.item_id', $item_id, '>') - ) - ) - // It is important that everything is indexed in order of changed date and - // then on entity_id because otherwise the conditions above will not match - // correctly - ->orderBy('sai.changed', 'ASC') - ->orderBy($alias, 'ASC') - ->orderBy('sai.item_id', 'ASC'); - - return $statement; - } - - /** - * {@inheritdoc} - */ - public function getRemainingItems($limit = -1) { - // Check if the index is enabled, writable and exists. - if ($this->index->status() && !$this->index->isReadOnly()) { - $statement = $this->getRemainingItemsQuery(); - // Check if the result set needs to be limited. - if ($limit > -1) { - // Limit the number of results to the given value. - $statement->range(0, $limit); - } - return $statement->execute()->fetchCol(); - } - return array(); - } - - /** - * {@inheritdoc} - */ - public function getIndexedItemsCount() { - // Check if the index is enabled and exists. - if ($this->index->status()) { - return $this->getTotalItemsCount() - $this->getRemainingItemsCount(); - } - return 0; - } - - /** - * {@inheritdoc} - */ - public function getRemainingItemsCount() { - // Check if the index is enabled, writable and exists. - if ($this->index->status()) { - $statement = $this->getRemainingItemsQuery(); - return (int) $statement->countQuery()->execute()->fetchField(); - } - return 0; - } - - /** - * {@inheritdoc} - */ - public function getTotalItemsCount() { - // Check if the index is enabled and exists. - if ($this->index->status()) { - return (int) $this->createSelectStatement() - ->countQuery() - ->execute() - ->fetchField(); - } - return 0; - } - - /** * {@inheritdoc} */ public function getViewModes() { diff --git a/lib/Drupal/search_api/Datasource/DatasourcePluginManager.php b/lib/Drupal/search_api/Datasource/DatasourcePluginManager.php index 59b77ac..9cde907 100644 --- a/lib/Drupal/search_api/Datasource/DatasourcePluginManager.php +++ b/lib/Drupal/search_api/Datasource/DatasourcePluginManager.php @@ -6,6 +6,7 @@ namespace Drupal\search_api\Datasource; +use Drupal\Component\Utility\String; use Drupal\Core\Plugin\DefaultPluginManager; use Drupal\Core\Cache\CacheBackendInterface; use Drupal\Core\Extension\ModuleHandlerInterface; @@ -37,4 +38,21 @@ class DatasourcePluginManager extends DefaultPluginManager { $this->alterInfo('search_api_datasource_info'); } + /** + * Get a list of plugin definition labels. + * + * @return array + * An associative array containing the plugin label, keyed by the plugin ID. + */ + public function getDefinitionLabels() { + // Initialize the options variable to an empty array. + $options = array(); + // Iterate through the datasource plugin definitions. + foreach ($this->getDefinitions() as $plugin_id => $plugin_definition) { + // Add the plugin to the list. + $options[$plugin_id] = String::checkPlain($plugin_definition['label']); + } + return $options; + } + } diff --git a/lib/Drupal/search_api/Entity/Index.php b/lib/Drupal/search_api/Entity/Index.php index c045420..ac9d5b0 100644 --- a/lib/Drupal/search_api/Entity/Index.php +++ b/lib/Drupal/search_api/Entity/Index.php @@ -9,7 +9,6 @@ namespace Drupal\search_api\Entity; use Drupal\Core\Cache\Cache; use Drupal\Core\Config\Entity\ConfigEntityBase; use Drupal\Core\Entity\EntityStorageInterface; -use Drupal\Core\Entity; use Drupal\Core\TypedData\ComplexDataDefinitionInterface; use Drupal\Core\TypedData\DataReferenceDefinitionInterface; use Drupal\Core\TypedData\ListDataDefinitionInterface; @@ -19,6 +18,7 @@ use Drupal\search_api\Processor\ProcessorInterface; use Drupal\search_api\Query\QueryInterface; use Drupal\search_api\Server\ServerInterface; use Drupal\search_api\Utility\Utility; +use Drupal\search_api\Datasource\DatasourceInterface; /** * Defines the search index configuration entity. @@ -158,6 +158,27 @@ class Index extends ConfigEntityBase implements IndexInterface { protected $datasourcePluginInstance; /** + * The tracker plugin ID. + * + * @var string + */ + public $trackerPluginId; + + /** + * The tracker plugin configuration. + * + * @var array + */ + public $trackerPluginConfig = array(); + + /** + * The tracker plugin instance. + * + * @var \Drupal\search_api\Tracker\TrackerInterface + */ + protected $trackerPluginInstance; + + /** * The machine name of the server on which data should be indexed. * * @var string @@ -196,8 +217,14 @@ class Index extends ConfigEntityBase implements IndexInterface { * {@inheritdoc} */ public function __construct(array $values, $entity_type) { + // Perform default instance construction. parent::__construct($values, $entity_type); - + // Check if the tracker plugin ID is unconfigured. + if ($this->trackerPluginId === NULL) { + // Set tracker plugin ID to the default tracker. + $this->trackerPluginId = \Drupal::config('search_api.settings')->get('default_tracker'); + } + // Merge in default options. $this->options += array( 'cron_limit' => \Drupal::configFactory()->get('search_api.settings')->get('cron_limit'), 'index_directly' => FALSE, @@ -208,8 +235,8 @@ class Index extends ConfigEntityBase implements IndexInterface { * Clones an index object. */ public function __clone() { - // Prevent the datasource and server instance from being cloned. - $this->datasourcePluginInstance = $this->server = NULL; + // Prevent the datasource, tracker and server instance from being cloned. + $this->datasourcePluginInstance = $this->trackerPluginInstance = $this->server = NULL; } /** @@ -294,14 +321,10 @@ class Index extends ConfigEntityBase implements IndexInterface { public function getDatasource() { // Check if the datasource plugin instance needs to be resolved. if (!$this->datasourcePluginInstance && $this->hasValidDatasource()) { - // Get the ID of the datasource plugin. - $datasource_plugin_id = $this->datasourcePluginId; - // Get the datasource plugin manager. - $datasource_plugin_manager = \Drupal::service('search_api.datasource.plugin.manager'); // Get the plugin configuration for the datasource. $datasource_plugin_configuration = array('index' => $this) + $this->datasourcePluginConfig; // Create a datasource plugin instance. - $this->datasourcePluginInstance = $datasource_plugin_manager->createInstance($datasource_plugin_id, $datasource_plugin_configuration); + $this->datasourcePluginInstance = \Drupal::service('search_api.datasource.plugin.manager')->createInstance($this->getDatasourceId(), $datasource_plugin_configuration); } return $this->datasourcePluginInstance; @@ -310,6 +333,37 @@ class Index extends ConfigEntityBase implements IndexInterface { /** * {@inheritdoc} */ + public function hasValidTracker() { + // Get the tracker plugin manager. + $tracker_plugin_definition = \Drupal::service('plugin.manager.search_api.tracker')->getDefinition($this->getTrackerId()); + // Determine whether the tracker is valid. + return !empty($tracker_plugin_definition); + } + + /** + * {@inheritdoc} + */ + public function getTrackerId() { + return $this->trackerPluginId; + } + + /** + * {@inheritdoc} + */ + public function getTracker() { + // Check if the tracker plugin instance needs to be resolved. + if (!$this->trackerPluginInstance && $this->hasValidTracker()) { + // Get the plugin configuration for the tracker. + $tracker_plugin_configuration = array('index' => $this) + $this->trackerPluginConfig; + // Create a tracker plugin instance. + $this->trackerPluginInstance = \Drupal::service('plugin.manager.search_api.tracker')->createInstance($this->getTrackerId(), $tracker_plugin_configuration); + } + return $this->trackerPluginInstance; + } + + /** + * {@inheritdoc} + */ public function hasValidServer() { return $this->serverMachineName !== NULL && \Drupal::entityManager()->getStorage('search_api_server')->load($this->serverMachineName) !== NULL; } @@ -701,27 +755,84 @@ class Index extends ConfigEntityBase implements IndexInterface { * {@inheritdoc} */ public function index($limit = '-1') { - $next_set = $this->getDatasource()->getRemainingItems($limit); - $items = $this->getDatasource()->loadMultiple($next_set); - $ids_indexed = $this->indexItems($items); - $this->getDatasource()->trackIndexed($ids_indexed); - return count($ids_indexed); + // Initialize the count variable to zero. + $count = 0; + // Check if a valid tracker and datasource is available. + if ($this->hasValidTracker() && $this->hasValidDatasource()) { + // Get the tracker. + $tracker = $this->getTracker(); + // Get the next set of items. + $next_set = $tracker->getRemainingItems(NULL, $limit); + // Iterate through the remaining items. + foreach ($next_set as $item_type => $ids) { + // Get the datasource for the current item type. + // @todo: Should be reworked once multiple datasources are supported. + $datasource = $this->getDatasource(); + // Load the items from the datasource. + $items = $datasource->loadMultiple($ids); + // Index the items. + $ids_indexed = $this->indexItems($items); + // Mark the indexed items. + $tracker->trackIndexed($datasource, $ids_indexed); + // Increment count by the number of indexed items. + $count += count($ids_indexed); + } + } + return $count; } /** * {@inheritdoc} */ public function reindex() { - return $this->getDatasource()->trackUpdate(); + // Check whether a valid tracker and datasource is available. + if ($this->hasValidTracker() && $this->hasValidDatasource()) { + // Mark all items for processing for the current datasource. + return $this->getTracker()->trackUpdated($this->getDatasource()); + } + return FALSE; + } + + /** + * {@inheritdoc} + */ + public function insertItems(DatasourceInterface $datasource, array $ids) { + return ($this->hasValidTracker() && $this->getTracker()->trackInserted($datasource, $ids)); + } + + /** + * {@inheritdoc} + */ + public function updateItems(DatasourceInterface $datasource, array $ids) { + return ($this->hasValidTracker() && $this->getTracker()->trackUpdated($datasource, $ids)); + } + + /** + * {@inheritdoc} + */ + public function deleteItems(DatasourceInterface $datasource, array $ids) { + // Remove the items from the tracker. + $success = ($this->hasValidTracker() && $this->getTracker()->trackDeleted($datasource, $ids)); + // Check whether a valid server is available. + if ($this->hasValidServer()) { + // Remove the items from the server. + $this->getServer()->deleteItems($this, $ids); + } + return $success; } /** * {@inheritdoc} */ public function clear() { - $this->getServer()->deleteAllItems($this); - $this->getDatasource()->trackUpdate(); - return TRUE; + // Check whether the index has a valid server and reindex was successful. + if ($this->reindex() && $this->hasValidServer()) { + // Remove all items for this index from the server. + $this->getServer()->deleteAllItems($this); + + return TRUE; + } + return FALSE; } /** @@ -781,7 +892,7 @@ class Index extends ConfigEntityBase implements IndexInterface { } } // Only check if there is a datasource - if ($this->getDatasource()) { + if ($this->hasValidDatasource()) { // If the index is new (so no update) and the status is enabled, start tracking // If the index is not new but the original status is disabled and the new status, start tracking if ((!$update && $this->status()) || ($update && ($this->status() && !$this->original->status()))) { @@ -815,18 +926,4 @@ class Index extends ConfigEntityBase implements IndexInterface { } } - /** - * {@inheritdoc} - */ - public function getLastIndexed() { - return \Drupal::state()->get($this->id() . '.last_indexed', array('changed' => '0', 'item_id' => '0')); - } - - /** - * {@inheritdoc} - */ - public function setLastIndexed($changed, $item_id) { - return \Drupal::state()->set($this->id() . '.last_indexed', array('changed' => $changed, 'item_id' => $item_id)); - } - } diff --git a/lib/Drupal/search_api/Form/IndexClearConfirmForm.php b/lib/Drupal/search_api/Form/IndexClearConfirmForm.php index d4b5eab..6a1f781 100644 --- a/lib/Drupal/search_api/Form/IndexClearConfirmForm.php +++ b/lib/Drupal/search_api/Form/IndexClearConfirmForm.php @@ -38,11 +38,8 @@ class IndexClearConfirmForm extends EntityConfirmFormBase { public function submit(array $form, array &$form_state) { // Get the search index entity object. $entity = $this->getEntity(); - // Check if the search index has a valid server available. - if ($entity->hasValidServer()) { - // Delete items from the server related to the current search index. - $entity->getServer()->deleteAllItems($entity); - } + // Clear the index. + $entity->clear(); // Redirect to the index view page. $form_state['redirect_route'] = array( 'route_name' => 'search_api.index_view', diff --git a/lib/Drupal/search_api/Form/IndexForm.php b/lib/Drupal/search_api/Form/IndexForm.php index d99c066..93c53b6 100644 --- a/lib/Drupal/search_api/Form/IndexForm.php +++ b/lib/Drupal/search_api/Form/IndexForm.php @@ -11,6 +11,7 @@ use Drupal\Core\Entity\EntityFormController; use Drupal\Core\Entity\EntityManager; use Drupal\search_api\Index\IndexInterface; use Drupal\search_api\Datasource\DatasourcePluginManager; +use Drupal\search_api\Tracker\TrackerPluginManager; use Drupal\Component\Utility\String; /** @@ -39,17 +40,30 @@ class IndexForm extends EntityFormController { protected $datasourcePluginManager; /** + * The Search API tracker plugin manager. + * + * This object members must be set to anything other than private in order + * for \Drupal\Core\DependencyInjection\DependencySerialization to detect. + * + * @var \Drupal\search_api\Tracker\TrackerPluginManager + */ + protected $trackerPluginManager; + + /** * Create an IndexForm object. * * @param \Drupal\Core\Entity\EntityManager $entity_manager * The entity manager. * @param \Drupal\search_api\Datasource\DatasourcePluginManager $datasource_plugin_manager * The search datasource plugin manager. + * @param \Drupal\search_api\Tracker\TrackerPluginManager $tracker_plugin_manager + * The Search API tracker plugin manager. */ - public function __construct(EntityManager $entity_manager, DatasourcePluginManager $datasource_plugin_manager) { + public function __construct(EntityManager $entity_manager, DatasourcePluginManager $datasource_plugin_manager, TrackerPluginManager $tracker_plugin_manager) { // Setup object members. $this->entityManager = $entity_manager; $this->datasourcePluginManager = $datasource_plugin_manager; + $this->trackerPluginManager = $tracker_plugin_manager; } /** @@ -73,21 +87,13 @@ class IndexForm extends EntityFormController { } /** - * Get a list of datasource plugin definitions for use with a select element. + * Get the Search API tracker plugin manager. * - * @return array - * An associative array of datasource plugin names, keyed by the datasource - * plugin ID. + * @return \Drupal\search_api\Tracker\TrackerPluginManager + * An instance of TrackerPluginManager. */ - protected function getDatasourcePluginDefinitionOptions() { - // Initialize the options variable to an empty array. - $options = array(); - // Iterate through the datasource plugin definitions. - foreach ($this->getDatasourcePluginManager()->getDefinitions() as $plugin_id => $plugin_definition) { - // Add the plugin to the list. - $options[$plugin_id] = String::checkPlain($plugin_definition['label']); - } - return $options; + protected function getTrackerPluginManager() { + return $this->trackerPluginManager; } /** @@ -134,7 +140,8 @@ class IndexForm extends EntityFormController { public static function create(ContainerInterface $container) { return new static( $container->get('entity.manager'), - $container->get('search_api.datasource.plugin.manager') + $container->get('search_api.datasource.plugin.manager'), + $container->get('plugin.manager.search_api.tracker') ); } @@ -199,8 +206,6 @@ class IndexForm extends EntityFormController { ), '#weight' => 2, ); - // Build the datasource element. - $options = $this->getDatasourcePluginDefinitionOptions(); // Check if the datasource plugin changed. if (!empty($form_state['values']['datasourcePluginId'])) { @@ -208,6 +213,12 @@ class IndexForm extends EntityFormController { drupal_set_message($this->t('Please configure the used data type.'), 'warning'); } + // Check if the datasource plugin changed. + if (!empty($form_state['values']['trackerPluginId'])) { + // Notify the user about the tracker configuration change. + drupal_set_message($this->t('Please configure the used tracker.'), 'warning'); + } + // Attach the admin css $form['#attached']['library'][] = 'search_api/drupal.search_api.admin_css'; @@ -215,7 +226,7 @@ class IndexForm extends EntityFormController { '#type' => 'select', '#title' => $this->t('Data type'), '#description' => $this->t('Select the data type of items that will be indexed in this index. E.g. should it be nodes content or files data. This setting cannot be changed afterwards.'), - '#options' => $options, + '#options' => $this->getDatasourcePluginManager()->getDefinitionLabels(), '#default_value' => $index->hasValidDatasource() ? $index->getDatasource()->getPluginId() : NULL, '#required' => TRUE, '#disabled' => !$index->isNew(), @@ -256,6 +267,51 @@ class IndexForm extends EntityFormController { // Build the datasource configuration form. $this->buildDatasourceConfigForm($form, $form_state, $index); + $form['trackerPluginId'] = array( + '#type' => 'select', + '#title' => $this->t('Tracker'), + '#description' => $this->t('Select the type of tracker which should be used for keeping track of item changes. This setting cannot be changed afterwards.'), + '#options' => $this->getTrackerPluginManager()->getDefinitionLabels(), + '#default_value' => $index->hasValidTracker() ? $index->getTracker()->getPluginId() : NULL, + '#required' => TRUE, + '#disabled' => !$index->isNew(), + '#ajax' => array( + 'trigger_as' => array('name' => 'trackerpluginid_configure'), + 'callback' => '\Drupal\search_api\Form\IndexForm::buildAjaxTrackerConfigForm', + 'wrapper' => 'search-api-tracker-config-form', + 'method' => 'replace', + 'effect' => 'fade', + ), + '#weight' => 6, + ); + + // Build the tracker plugin configuration container element. + $form['trackerPluginConfig'] = array( + '#type' => 'container', + '#attributes' => array( + 'id' => 'search-api-tracker-config-form', + ), + '#weight' => 7, + '#tree' => TRUE, + ); + + $form['trackerConfigureButton'] = array( + '#type' => 'submit', + '#name' => 'trackerpluginid_configure', + '#value' => t('Configure'), + '#limit_validation_errors' => array(array('trackerPluginId')), + '#submit' => array('\Drupal\search_api\Form\IndexForm::submitAjaxTrackerConfigForm'), + '#ajax' => array( + 'callback' => '\Drupal\search_api\Form\IndexForm::buildAjaxTrackerConfigForm', + 'wrapper' => 'search-api-tracker-config-form', + ), + '#weight' => 8, + '#attributes' => array('class' => array('js-hide')), + ); + + // Build the tracker configuration form. + $this->buildTrackerConfigForm($form, $form_state, $index); + // Build the server machine name element. $form['serverMachineName'] = array( '#type' => 'select', @@ -263,7 +319,7 @@ class IndexForm extends EntityFormController { '#description' => $this->t('Select the server this index should reside on. Index can not be enabled without connection to valid server.'), '#options' => array('' => $this->t('< No server >')) + $this->getServerOptions(), '#default_value' => $index->hasValidServer() ? $index->getServer()->id() : NULL, - '#weight' => 6, + '#weight' => 9, ); // Build the status element. @@ -279,7 +335,7 @@ class IndexForm extends EntityFormController { '[name="serverMachineName"]' => array('value' => '') ), ), - '#weight' => 7, + '#weight' => 10, ); // Build the description element. @@ -288,7 +344,7 @@ class IndexForm extends EntityFormController { '#title' => $this->t('Description'), '#description' => $this->t('Enter a description for the index.'), '#default_value' => $index->getDescription(), - '#weight' => 8, + '#weight' => 11, ); $form['options'] = array( @@ -296,7 +352,7 @@ class IndexForm extends EntityFormController { '#type' => 'details', '#title' => t('Index options'), '#collapsed' => TRUE, - '#weight' => 9, + '#weight' => 12, ); // Build the read only element. @@ -307,7 +363,7 @@ class IndexForm extends EntityFormController { '#default_value' => $index->isReadOnly(), // changed #parents so this option is not saved into options '#parents' => array('readOnly'), - '#weight' => 10, + '#weight' => 13, ); // Build the index directly element. $form['options']['index_directly'] = array( @@ -315,7 +371,7 @@ class IndexForm extends EntityFormController { '#title' => $this->t('Index items immediately'), '#description' => $this->t('Immediately index new or updated items instead of waiting for the next cron run. This might have serious performance drawbacks and is generally not advised for larger sites.'), '#default_value' => $index->getOption('index_directly'), - '#weight' => 11, + '#weight' => 14, ); // Build the cron limit element. $form['options']['cron_limit'] = array( @@ -329,7 +385,7 @@ class IndexForm extends EntityFormController { ':input[name="options[index_directly]"]' => array('checked' => TRUE), ), ), - '#weight' => 12, + '#weight' => 15, ); } @@ -372,6 +428,43 @@ class IndexForm extends EntityFormController { } /** + * Build the tracker configuration form. + * + * @param array $form + * An associative array containing the structure of the form. + * @param array $form_state + * An associative array containing the current state of the form. + * @param \Drupal\search_api\Index\IndexInterface $server + * An instance of IndexInterface. + */ + public function buildTrackerConfigForm(array &$form, array &$form_state, IndexInterface $index) { + // Check if the index has a valid tracker configured. + if ($index->hasValidTracker()) { + // Get the tracker. + $tracker = $index->getTracker(); + // Get the tracker plugin definition. + $tracker_plugin_definition = $tracker->getPluginDefinition(); + // Build the tracker configuration form. + if (($tracker_plugin_config_form = $tracker->buildConfigurationForm(array(), $form_state))) { + // Modify the tracker plugin configuration container element. + $form['trackerPluginConfig']['#type'] = 'details'; + $form['trackerPluginConfig']['#title'] = $this->t('Configure @plugin', array('@plugin' => $tracker_plugin_definition['label'])); + $form['trackerPluginConfig']['#description'] = String::checkPlain($tracker_plugin_definition['description']); + $form['trackerPluginConfig']['#open'] = $index->isNew() ? TRUE : $index->isNew(); + + // Attach the build tracker plugin configuration form. + $form['trackerPluginConfig'] += $tracker_plugin_config_form; + } + } + // Do not notify the user about a missing tracker plugin if a new index + // is being configured. + elseif (!$index->isNew()) { + // Notify the user about the missing tracker plugin. + drupal_set_message($this->t('The tracker plugin is missing or invalid.'), 'error'); + } + } + + /** * Button submit handler for datasource configure button 'datasource_configure' button. */ public static function submitAjaxDatasourceConfigForm($form, &$form_state) { @@ -379,6 +472,13 @@ class IndexForm extends EntityFormController { } /** + * Button submit handler for tracker configure button 'tracker_configure' button. + */ + public static function submitAjaxTrackerConfigForm($form, &$form_state) { + $form_state['rebuild'] = TRUE; + } + + /** * Build the datasource plugin configuration form in context of an Ajax * request. * @@ -396,6 +496,22 @@ class IndexForm extends EntityFormController { } /** + * Build the tracker plugin configuration form in context of an Ajax request. + * + * @param array $form + * An associative array containing the structure of the form. + * @param array $form_state + * An associative array containing the current state of the form. + * + * @return array + * An associative array containing the structure of the form. + */ + public static function buildAjaxTrackerConfigForm(array $form, array &$form_state) { + // Get the tracker plugin configuration form. + return $form['trackerPluginConfig']; + } + + /** * {@inheritdoc} */ public function validate(array $form, array &$form_state) { @@ -427,6 +543,33 @@ class IndexForm extends EntityFormController { $datasource_form_state['values'] = $form_state['values']['datasourcePluginConfig']; $entity->getDatasource()->validateConfigurationForm($form['datasourcePluginConfig'], $datasource_form_state); } + // Get the current tracker plugin ID. + $tracker_plugin_id = $entity->hasValidTracker() ? $entity->getTracker()->getPluginId() : NULL; + // Check if the tracker plugin changed. + if ($tracker_plugin_id !== $form_state['values']['trackerPluginId']) { + // Check if the tracker plugin configuration form input values exist. + if (!empty($form_state['input']['trackerPluginConfig'])) { + // Overwrite the plugin configuration form input values with an empty + // array. This will force the Drupal Form API to use the default values. + $form_state['input']['trackerPluginConfig'] = array(); + } + // Check if the tracker plugin configuration form values exist. + if (!empty($form_state['values']['trackerPluginConfig'])) { + // Overwrite the plugin configuration form values with an empty array. + // This has no effect on the Drupal Form API but is done to keep the + // data consistent. + $form_state['values']['trackerPluginConfig'] = array(); + } + } + // Check if the entity has a valid tracker plugin. + elseif ($entity->hasValidTracker()) { + // Build the tracker plugin configuration form state. + $tracker_form_state = array( + 'values' => $form_state['values']['trackerPluginConfig'], + ); + // Validate the tracker plugin configuration form. + $entity->getTracker()->validateConfigurationForm($form['trackerPluginConfig'], $tracker_form_state); + } } /** @@ -446,20 +589,37 @@ class IndexForm extends EntityFormController { //$entity->options['fields'] = $entity->getDatasource()->getEntityType()->get('search_api_default_fields'); } // Check if the entity has a valid datasource plugin. - elseif ($entity->hasValidDatasource()) { - // Get the datasource from the entity. - $datasource = $entity->getDatasource(); - $datasource_form_state['values'] = $form_state['values']['datasourcePluginConfig']; - // Also add all additional values so that we can read them in the plugin. Useful for the status - $indexValues = $form_state['values']; - unset($indexValues['datasourcePluginConfig']); - $datasource_form_state['values']['index'] = $indexValues; - + if ($entity->hasValidDatasource()) { + // Build the datasource plugin configuration form state. + $datasource_form_state = array( + 'values' => $form_state['values']['datasourcePluginConfig'], + ); + // Also add all additional values so that we can read them in the plugin. + // Useful for the status + $datasource_form_state['values']['index'] = $form_state['values']; + // Remove the datasource plugin configuration from the index form state + // as it is already provided. + unset($datasource_form_state['values']['index']['datasourcePluginConfig']); // Submit the datasource plugin configuration form. - $datasource->submitConfigurationForm($form['datasourcePluginConfig'], $datasource_form_state); + $entity->getDatasource()->submitConfigurationForm($form['datasourcePluginConfig'], $datasource_form_state); + } + // Check if the entity has a valid tracker plugin. + if ($entity->hasValidTracker()) { + // Build the tracker plugin configuration form state. + $tracker_form_state = array( + 'values' => $form_state['values']['trackerPluginConfig'] + ); + // Also add all additional values so that we can read them in the plugin. + // Useful for the status. + $tracker_form_state['values']['index'] = $form_state['values']; + // Remove the tracker plugin configuration from the index form state + // as it is already provided. + unset($tracker_form_state['values']['index']['trackerPluginConfig']); + // Submit the tracker plugin configuration form. + $entity->getTracker()->submitConfigurationForm($form['trackerPluginConfig'], $tracker_form_state); } - $entity = parent::submit($form, $form_state); - return $entity; + + return parent::submit($form, $form_state); } /** diff --git a/lib/Drupal/search_api/Form/IndexStatusForm.php b/lib/Drupal/search_api/Form/IndexStatusForm.php index 90ddb80..f7763f2 100644 --- a/lib/Drupal/search_api/Form/IndexStatusForm.php +++ b/lib/Drupal/search_api/Form/IndexStatusForm.php @@ -29,10 +29,8 @@ class IndexStatusForm extends FormBase { public function buildForm(array $form, array &$form_state, IndexInterface $index = NULL) { // Attach the search index to the form. $form['#index'] = $index; - // Check if the index has a valid datasource available. - if ($index->hasValidDatasource()) { - // Get the datasource used by the search index. - $datasource = $index->getDatasource(); + // Check if the index has a valid tracker available. + if ($index->hasValidTracker()) { // Build the index now option. $form['index'] = array( '#type' => 'details', @@ -43,7 +41,7 @@ class IndexStatusForm extends FormBase { ), ); // Determine whether the index has remaining items to index. - $has_remaining_items = ($datasource->getRemainingItemsCount() > 0); + $has_remaining_items = ($index->getTracker()->getRemainingItemsCount() > 0); // Get the value which represent indexing all remaining items. $all_value = $this->t('all', array(), array('context' => 'items to index')); // Build the number of batches to execute. diff --git a/lib/Drupal/search_api/Form/ServerClearConfirmForm.php b/lib/Drupal/search_api/Form/ServerClearConfirmForm.php index 1037b01..2549e38 100644 --- a/lib/Drupal/search_api/Form/ServerClearConfirmForm.php +++ b/lib/Drupal/search_api/Form/ServerClearConfirmForm.php @@ -51,12 +51,8 @@ class ServerClearConfirmForm extends EntityConfirmFormBase { $ignored_indexes = array(); // Iterate through the attached indexes. foreach ($entity->getIndexes() as $index) { - // Check if reindexing items is possible. - if ($index->reindex()) { - // Delete all indexed data for the current index. - $entity->deleteAllItems($index); - } - else { + // Check whether clearing the index was successful. + if (!$index->clear()) { // Add the index label to the list of ignored or failed indexes. $ignored_indexes[] = l($index->label(), $index->getSystemPath('canonical')); } diff --git a/lib/Drupal/search_api/Index/IndexInterface.php b/lib/Drupal/search_api/Index/IndexInterface.php index 8ebc96c..f0bfb19 100644 --- a/lib/Drupal/search_api/Index/IndexInterface.php +++ b/lib/Drupal/search_api/Index/IndexInterface.php @@ -9,6 +9,7 @@ namespace Drupal\search_api\Index; use Drupal\Core\Config\Entity\ConfigEntityInterface; use Drupal\search_api\Query\QueryInterface; use Drupal\search_api\Server\ServerInterface; +use Drupal\search_api\Datasource\DatasourceInterface; /** * Defines the interface for index entities. @@ -26,7 +27,7 @@ interface IndexInterface extends ConfigEntityInterface { /** * Determine whether this index is read-only. * - * @return boolean + * @return bool * TRUE if this index is read-only, otherwise FALSE. */ public function isReadOnly(); @@ -85,7 +86,7 @@ interface IndexInterface extends ConfigEntityInterface { /** * Determine whether the datasource is valid. * - * @return boolean + * @return bool * TRUE if the datasource is valid, otherwise FALSE. */ public function hasValidDatasource(); @@ -107,9 +108,33 @@ interface IndexInterface extends ConfigEntityInterface { public function getDatasource(); /** + * Determine whether the tracker is valid. + * + * @return bool + * TRUE if the tracker is valid, otherwise FALSE. + */ + public function hasValidTracker(); + + /** + * Retrieves the tracker plugin's ID. + * + * @return string + * The ID of the tracker plugin used by this index. + */ + public function getTrackerId(); + + /** + * Retrieves the tracker plugin. + * + * @return \Drupal\search_api\Tracker\TrackerInterface + * An instance of TrackerInterface. + */ + public function getTracker(); + + /** * Determines whether the server is valid. * - * @return boolean + * @return bool * TRUE if the server is valid, otherwise FALSE. */ public function hasValidServer(); @@ -270,6 +295,45 @@ interface IndexInterface extends ConfigEntityInterface { public function reindex(); /** + * Add items from a specific datasource to the index. + * + * @param \Drupal\search_api\Datasource\DatasourceInterface $datasource + * An instance of DatasourceInterface which manages the items. + * @param array $ids + * An array of item IDs. + * + * @return bool + * TRUE if the operation was successful, otherwise FALSE. + */ + public function insertItems(DatasourceInterface $datasource, array $ids); + + /** + * Update items from a specific datasource present in the index. + * + * @param \Drupal\search_api\Datasource\DatasourceInterface $datasource + * An instance of DatasourceInterface which manages the items. + * @param array $ids + * An array of item IDs. + * + * @return bool + * TRUE if the operation was successful, otherwise FALSE. + */ + public function updateItems(DatasourceInterface $datasource, array $ids); + + /** + * Delete items from the index. + * + * @param \Drupal\search_api\Datasource\DatasourceInterface $datasource + * An instance of DatasourceInterface which manages the items. + * @param array $ids + * An array of items IDs. + * + * @return bool + * TRUE if the operation was successful, otherwise FALSE. + */ + public function deleteItems(DatasourceInterface $datasource, array $ids); + + /** * Clears all items in this index and marks them for reindexing. * * @return bool @@ -298,26 +362,4 @@ interface IndexInterface extends ConfigEntityInterface { */ public function query(array $options = array()); - /** - * Get last indexed state for this index. - * - * @return array - * An array containing the last indexed state for this index. Format is - * { 'changed' => 0, 'item_id' => 1 } - * - */ - public function getLastIndexed(); - - /** - * Set last indexed state for this index. - * - * @param $changed - * The last timestamp that was indexed - * @param $item_id - * The last item that was indexed - * @return array - * An array containing the stored last indexed state for this index. Format is - * { 'changed' => 0, 'item_id' => 1 } - */ - public function setLastIndexed($changed, $item_id); } diff --git a/lib/Drupal/search_api/Plugin/SearchApi/Datasource/ContentEntityDatasource.php b/lib/Drupal/search_api/Plugin/SearchApi/Datasource/ContentEntityDatasource.php index 60f7655..a1039da 100644 --- a/lib/Drupal/search_api/Plugin/SearchApi/Datasource/ContentEntityDatasource.php +++ b/lib/Drupal/search_api/Plugin/SearchApi/Datasource/ContentEntityDatasource.php @@ -10,7 +10,6 @@ use Drupal\Core\Entity\EntityInterface; use Drupal\Core\Plugin\ContainerFactoryPluginInterface; use Drupal\Core\TypedData\ComplexDataInterface; use Symfony\Component\DependencyInjection\ContainerInterface; -use Drupal\Core\Database\Connection; use Drupal\Core\Entity\EntityManager; use Drupal\search_api\Datasource\DatasourcePluginBase; use Drupal\Component\Utility\String; @@ -49,19 +48,10 @@ class ContentEntityDatasource extends DatasourcePluginBase implements ContainerF protected $typedDataManager; /** - * The database connection. - * - * @var \Drupal\Core\Database\Connection - */ - protected $databaseConnection; - - /** * Create a ContentEntityDatasource object. * * @param \Drupal\Core\Entity\EntityManager $entity_manager * The entity manager. - * @param \Drupal\Core\Database\Connection $connection - * A connection to the database. * @param array $configuration * A configuration array containing information about the plugin instance. * @param string $plugin_id @@ -69,14 +59,13 @@ class ContentEntityDatasource extends DatasourcePluginBase implements ContainerF * @param array $plugin_definition * The plugin implementation definition. */ - public function __construct(EntityManager $entity_manager, Connection $connection, array $configuration, $plugin_id, array $plugin_definition) { + public function __construct(EntityManager $entity_manager, array $configuration, $plugin_id, array $plugin_definition) { // Initialize the parent chain of objects. - parent::__construct($connection, $configuration, $plugin_id, $plugin_definition); + parent::__construct($configuration, $plugin_id, $plugin_definition); // Setup object members. $this->entityManager = $entity_manager; $this->storage = $entity_manager->getStorage($plugin_definition['entity_type']); $this->typedDataManager = \Drupal::typedDataManager(); - $this->databaseConnection = $connection; } /** @@ -85,9 +74,8 @@ class ContentEntityDatasource extends DatasourcePluginBase implements ContainerF public static function create(ContainerInterface $container, array $configuration, $plugin_id, $plugin_definition) { /** @var $entity_manager \Drupal\Core\Entity\EntityManager */ $entity_manager = $container->get('entity.manager'); - /** @var $connection \Drupal\Core\Database\Connection */ - $connection = $container->get('database'); - return new static($entity_manager, $connection, $configuration, $plugin_id, $plugin_definition); + + return new static($entity_manager, $configuration, $plugin_id, $plugin_definition); } /** @@ -108,17 +96,21 @@ class ContentEntityDatasource extends DatasourcePluginBase implements ContainerF * {@inheritdoc} */ public function load($id) { - return $this->storage->load($id); + // Load the item by ID. + $items = $this->loadMultiple(array($id)); + return ($items) ? reset($items) : NULL; } /** * {@inheritdoc} */ public function loadMultiple(array $ids) { + // Load the items from storage. $items = $this->storage->loadMultiple($ids); // If we were unable to delete some of the items, mark them as deleted. if ($diff = array_diff_key(array_flip($ids), $items)) { - $this->trackDelete(array_keys($diff)); + // Remove the items from the index. + $this->getIndex()->deleteItems($this, array_keys($diff)); } return $items; } @@ -236,9 +228,10 @@ class ContentEntityDatasource extends DatasourcePluginBase implements ContainerF * {@inheritdoc} */ public function startTracking() { - $entity_ids = $this->getEntityIds(); - if (!empty($entity_ids)) { - $this->trackInsert($entity_ids); + // Check whether there are entities which need to be inserted. + if (($entity_ids = $this->getEntityIds())) { + // Register entities with the tracker. + $this->getIndex()->insertItems($this, $entity_ids); } } @@ -246,9 +239,10 @@ class ContentEntityDatasource extends DatasourcePluginBase implements ContainerF * {@inheritdoc} */ public function stopTracking() { - $entity_ids = $this->getEntityIds(); - if (!empty($entity_ids)) { - $this->trackDelete($entity_ids); + // Check whether there are entities which need to be removed. + if (($entity_ids = $this->getEntityIds())) { + // Remove the items from the tracker. + $this->getIndex()->deleteItems($this, $entity_ids); } } @@ -256,9 +250,10 @@ class ContentEntityDatasource extends DatasourcePluginBase implements ContainerF * {@inheritdoc} */ public function startTrackingBundles(array $bundles) { - $entity_ids = $this->getEntityIds($bundles); - if (!empty($entity_ids)) { - return $this->trackInsert($entity_ids); + // Check whether there are entities which need to be inserted. + if (($entity_ids = $this->getEntityIds($bundles))) { + // Register entities with the tracker. + $this->getIndex()->insertItems($this, $entity_ids); } } @@ -266,9 +261,10 @@ class ContentEntityDatasource extends DatasourcePluginBase implements ContainerF * {@inheritdoc} */ public function stopTrackingBundles(array $bundles) { - $entity_ids = $this->getEntityIds($bundles); - if (!empty($entity_ids)) { - return $this->trackDelete($entity_ids); + // Check whether there are entities which need to be removed. + if (($entity_ids = $this->getEntityIds($bundles))) { + // Remove the items from the tracker. + $this->getIndex()->deleteItems($this, $entity_ids); } } @@ -298,7 +294,7 @@ class ContentEntityDatasource extends DatasourcePluginBase implements ContainerF /** * Determine whether the index is valid for this datasource. * - * @return boolean + * @return bool * TRUE if the index is valid, otherwise FALSE. */ protected function isValidIndex() { diff --git a/lib/Drupal/search_api/Plugin/SearchApi/Service/TestService.php b/lib/Drupal/search_api/Plugin/SearchApi/Service/TestService.php index eb6259a..02767ba 100644 --- a/lib/Drupal/search_api/Plugin/SearchApi/Service/TestService.php +++ b/lib/Drupal/search_api/Plugin/SearchApi/Service/TestService.php @@ -20,14 +20,14 @@ class TestService extends ServicePluginBase implements ServiceExtraInfoInterface * {@inheritdoc} */ public function indexItems(IndexInterface $index, array $items) { - return array(); + return array_keys($items); } /** * {@inheritdoc} */ public function deleteItems(IndexInterface $index, array $ids) { - return array(); + return $ids; } /** diff --git a/lib/Drupal/search_api/Plugin/SearchApi/Service/TestService2.php b/lib/Drupal/search_api/Plugin/SearchApi/Service/TestService2.php index 1b37252..b2c5e5d 100644 --- a/lib/Drupal/search_api/Plugin/SearchApi/Service/TestService2.php +++ b/lib/Drupal/search_api/Plugin/SearchApi/Service/TestService2.php @@ -38,14 +38,14 @@ class TestService2 extends ServicePluginBase { * {@inheritdoc} */ public function indexItems(IndexInterface $index, array $items) { - return array(); + return array_keys($items); } /** * {@inheritdoc} */ public function deleteItems(IndexInterface $index, array $ids) { - return array(); + return $ids; } /** diff --git a/lib/Drupal/search_api/Plugin/SearchApi/Tracker/DefaultTracker.php b/lib/Drupal/search_api/Plugin/SearchApi/Tracker/DefaultTracker.php new file mode 100644 index 0000000..b85d6d4 --- /dev/null +++ b/lib/Drupal/search_api/Plugin/SearchApi/Tracker/DefaultTracker.php @@ -0,0 +1,426 @@ +connection = $connection; + } + + /** + * Get the database connection. + * + * @return \Drupal\Core\Database\Connection + * An instance of Connection. + */ + public function getDatabaseConnection() { + return $this->connection; + } + + /** + * {@inheritdoc} + */ + public static function create(ContainerInterface $container, array $configuration, $plugin_id, $plugin_definition) { + /** @var $connection \Drupal\Core\Database\Connection */ + $connection = $container->get('database'); + // Create the plugin instance. + return new static($connection, $configuration, $plugin_id, $plugin_definition); + } + + /** + * Creates a select statement. + * + * @return \Drupal\Core\Database\Query\SelectInterface + * An instance of SelectInterface. + */ + protected function createSelectStatement() { + return $this->getDatabaseConnection()->select('search_api_item', 'sai') + ->condition('index_id', $this->getIndex()->id()); + } + + /** + * Creates an insert statement. + * + * @return \Drupal\Core\Database\Query\Insert + * An instance of Insert. + */ + protected function createInsertStatement() { + return $this->getDatabaseConnection()->insert('search_api_item') + ->fields(array('item_id', 'index_id', 'changed')); + } + + /** + * Creates an update statement. + * + * @return \Drupal\Core\Database\Query\Update + * An instance of Update. + */ + protected function createUpdateStatement() { + return $this->getDatabaseConnection()->update('search_api_item') + ->condition('index_id', $this->getIndex()->id()); + } + + /** + * Creates a delete statement. + * + * @return \Drupal\Core\Database\Query\Delete + * An instance of ConditionInterface. + */ + protected function createDeleteStatement() { + return $this->getDatabaseConnection()->delete('search_api_item') + ->condition('index_id', $this->getIndex()->id()); + } + + /** + * Creates a statement which filters on the remaining items. + * + * @return \Drupal\Core\Database\Query\Select + * An instance of Select. + */ + protected function createRemainingItemsStatement() { + // Build the select statement. + // @todo: Should filter on the datasource once multiple datasources are + // supported. + $statement = $this->createSelectStatement(); + // Only the item ID is needed. + $statement->fields('sai', array('item_id')); + // Exclude items marked as indexed. + $statement->condition('sai.changed', 0, '>'); + // Sort items by changed timestamp. + $statement->orderBy('sai.changed', 'ASC'); + + return $statement; + } + + /** + * {@inheritdoc} + */ + public function trackInserted(DatasourceInterface $datasource, array $ids) { + // Initialize the success variable to FALSE. + $success = FALSE; + // Get the index. + $index = $this->getIndex(); + // Check if the index is enabled and writable. + if (!$index->isNew() && $index->status() && !$index->isReadOnly()) { + // Start a database transaction. + $transaction = $this->getDatabaseConnection()->startTransaction(); + // Catch any exception that may occur during insert. + try { + // Get the index ID. + $index_id = $index->id(); + // Iterate through the IDs in chunks of 1000 items. + foreach (array_chunk($ids, 1000) as $ids_chunk) { + // Build the insert statement. + $statement = $this->createInsertStatement(); + // Iterate through the chunked IDs. + foreach ($ids_chunk as $item_id) { + // Add the item ID to the insert statement. + // @todo: Once multiple datasource are support we should include it. + $statement->values(array( + 'item_id' => $item_id, + 'index_id' => $index_id, + 'changed' => REQUEST_TIME, + )); + } + // Execute the statement. + $statement->execute(); + } + // Mark operation as successful. + $success = TRUE; + } + catch (Exception $ex) { + // Log exception to watchdog. + watchdog_exception('Search API', $ex); + // Rollback any changes made to the database. + $transaction->rollback(); + } + } + return $success; + } + + /** + * {@inheritdoc} + */ + public function trackUpdated(DatasourceInterface $datasource, array $ids = NULL) { + // Initialize the success variable to FALSE. + $success = FALSE; + // Get the index. + $index = $this->getIndex(); + // Check if the index is enabled and writable. + if (!$index->isNew() && $index->status() && !$index->isReadOnly()) { + // Start a database transaction. + $transaction = $this->getDatabaseConnection()->startTransaction(); + // Catch any exception that may occur during insert. + try { + // Group the item IDs in chunks of maximum 1000 entries. + $ids_chunks = ($ids !== NULL ? array_chunk($ids, 1000) : array(NULL)); + // Iterate through the IDs in chunks of 1000 items. + foreach ($ids_chunks as $ids_chunk) { + // Build the update statement. + // @todo: Should include the datasource as filter once multiple + // datasources are supported. + $statement = $this->createUpdateStatement(); + // Set changed value to current REQUEST_TIME. + $statement->fields(array('changed' => REQUEST_TIME)); + // Check whether specific items should be updated. + if ($ids_chunk) { + $statement->condition('item_id', $ids_chunk); + } + // Execute the statement. + $statement->execute(); + } + // Mark operation as successful. + $success = TRUE; + } + catch (Exception $ex) { + // Log exception to watchdog. + watchdog_exception('Search API', $ex); + // Rollback any changes made to the database. + $transaction->rollback(); + } + } + return $success; + } + + /** + * {@inheritdoc} + */ + public function trackIndexed(DatasourceInterface $datasource, array $ids) { + // Initialize the success variable to FALSE. + $success = FALSE; + // Get the index. + $index = $this->getIndex(); + // Check if the index is enabled and writable. + if (!$index->isNew() && $index->status() && !$index->isReadOnly()) { + // Start a database transaction. + $transaction = $this->getDatabaseConnection()->startTransaction(); + // Catch any exception that may occur during insert. + try { + // Group the item IDs in chunks of maximum 1000 entries. + $ids_chunks = ($ids !== NULL ? array_chunk($ids, 1000) : array(NULL)); + // Iterate through the IDs in chunks of 1000 items. + foreach ($ids_chunks as $ids_chunk) { + // Build the update statement. + // @todo: Should include the datasource as filter once multiple + // datasources are supported. + $statement = $this->createUpdateStatement(); + // Set changed value to 0 which idicates the item was indexed. + $statement->fields(array('changed' => 0)); + // Ensure only specified items get marked as indexed. + $statement->condition('item_id', $ids_chunk); + // Execute the statement. + $statement->execute(); + } + // Mark operation as successful. + $success = TRUE; + } + catch (Exception $ex) { + // Log exception to watchdog. + watchdog_exception('Search API', $ex); + // Rollback any changes made to the database. + $transaction->rollback(); + } + } + return $success; + } + + /** + * {@inheritdoc} + */ + public function trackDeleted(DatasourceInterface $datasource, array $ids = NULL) { + // Initialize the success variable to FALSE. + $success = FALSE; + // Get the index. + $index = $this->getIndex(); + // Check if the index is enabled and writable. + if (!$index->isNew() && $index->status() && !$index->isReadOnly()) { + // Start a database transaction. + $transaction = $this->getDatabaseConnection()->startTransaction(); + // Catch any exception that may occur during insert. + try { + // Only perform the delete statement if at lease one ID is present or + // all items should be deleted. + if ($ids === NULL || $ids) { + // Group the item IDs in chunks of maximum 1000 entries. + $ids_chunks = ($ids !== NULL ? array_chunk($ids, 1000) : array(NULL)); + // Iterate through the IDs in chunks of 1000 items. + foreach ($ids_chunks as $ids_chunk) { + // Build the delete statement. + // @todo: Should include the datasource as filter once multiple + // datasources are supported. + $statement = $this->createDeleteStatement(); + // Check whether specific items should be removed. + if ($ids_chunk) { + $statement->condition('item_id', $ids_chunk); + } + // Execute the statement. + $statement->execute(); + } + } + // Mark operation as successful. + $success = TRUE; + } + catch (Exception $ex) { + // Log exception to watchdog. + watchdog_exception('Search API', $ex); + // Rollback any changes made to the database. + $transaction->rollback(); + } + } + return $success; + } + + /** + * {@inheritdoc} + */ + public function getRemainingItems(DatasourceInterface $datasource = NULL, $limit = -1) { + // Get the index. + $index = $this->getIndex(); + // Check if the index is enabled and writable. + if (!$index->isNew() && $index->status() && !$index->isReadOnly()) { + // Create the remaining items statement. + // @todo: Should include the datasource once multiple datasources are + // supported. + $statement = $this->createRemainingItemsStatement(); + // Check whether a range should be applied. + if ($limit > -1) { + $statement->range(0, $limit); + } + // @todo: Default is temporarly used because multiple datasources + // are currently not supported and ignored. + return array('default' => $statement->execute()->fetchCol()); + } + return array(); + } + + /** + * {@inheritdoc} + */ + public function getRemainingItemsCount(DatasourceInterface $datasource = NULL) { + // Get the index. + $index = $this->getIndex(); + // Check whether the index is enabled. + if (!$index->isNew() && $index->status()) { + // @todo: Should include the datasource as filter once multiple + // datasources are supported. + return (int) $this->createRemainingItemsStatement() + ->countQuery() + ->execute() + ->fetchField(); + } + return 0; + } + + /** + * {@inheritdoc} + */ + public function getTotalItemsCount(DatasourceInterface $datasource = NULL) { + // Get the index. + $index = $this->getIndex(); + // Check whether the index is enabled. + if (!$index->isNew() && $index->status()) { + // @todo: Should include the datasource as filter once multiple + // datasources are supported. + return (int) $this->createSelectStatement() + ->countQuery() + ->execute() + ->fetchField(); + } + return 0; + } + + /** + * {@inheritdoc} + */ + public function getIndexedItemsCount(DatasourceInterface $datasource = NULL) { + // Get the index. + $index = $this->getIndex(); + // Check whether the index is enabled. + if (!$index->isNew() && $index->status()) { + // Create the select statement. @todo: Should include the datasource once + // multiple datasources are supported. + $statement = $this->createSelectStatement(); + // Filter on indexed items. + $statement->condition('sai.changed', 0); + // Get the number of indexed items. + return (int) $statement + ->countQuery() + ->execute() + ->fetchField(); + } + return 0; + } + + /** + * {@inheritdoc} + */ + public function clear() { + // Initialize the success variable to FALSE. + $success = FALSE; + // Get the index. + $index = $this->getIndex(); + // Check if the index is enabled and writable. + if (!$index->isNew() && $index->status() && !$index->isReadOnly()) { + // Start a database transaction. + $transaction = $this->getDatabaseConnection()->startTransaction(); + // Catch any exception that may occur during insert. + try { + // Remove all items for the current index. + $this->createDeleteStatement()->execute(); + // Mark operation as successful. + $success = TRUE; + } + catch (Exception $ex) { + // Log exception to watchdog. + watchdog_exception('Search API', $ex); + // Rollback any changes made to the database. + $transaction->rollback(); + } + } + return $success; + } + +} diff --git a/lib/Drupal/search_api/Processor/ProcessorInterface.php b/lib/Drupal/search_api/Processor/ProcessorInterface.php index 6b0e3a8..00562aa 100644 --- a/lib/Drupal/search_api/Processor/ProcessorInterface.php +++ b/lib/Drupal/search_api/Processor/ProcessorInterface.php @@ -38,7 +38,7 @@ interface ProcessorInterface extends IndexPluginInterface { * @param \Drupal\search_api\Index\IndexInterface $index * The index to check for. * - * @return boolean + * @return bool * TRUE if the processor can run on the given index; FALSE otherwise. */ public static function supportsIndex(IndexInterface $index); diff --git a/lib/Drupal/search_api/Tracker/TrackerInterface.php b/lib/Drupal/search_api/Tracker/TrackerInterface.php new file mode 100644 index 0000000..3ea0585 --- /dev/null +++ b/lib/Drupal/search_api/Tracker/TrackerInterface.php @@ -0,0 +1,139 @@ +getIndex()->id() !== $datasource->getIndex()->id()) { + throw new SearchApiException('Cannot track datatsource due to index mismatch'); + } + } + +} diff --git a/lib/Drupal/search_api/Tracker/TrackerPluginManager.php b/lib/Drupal/search_api/Tracker/TrackerPluginManager.php new file mode 100644 index 0000000..385a6dc --- /dev/null +++ b/lib/Drupal/search_api/Tracker/TrackerPluginManager.php @@ -0,0 +1,59 @@ +setCacheBackend($cache_backend, $language_manager, 'search_api_trackers'); + $this->alterInfo('search_api_tracker_info'); + } + + /** + * Get a list of plugin definition labels. + * + * @return array + * An associative array containing the plugin label, keyed by the plugin ID. + */ + public function getDefinitionLabels() { + // Initialize the options variable to an empty array. + $options = array(); + // Iterate through the tracker plugin definitions. + foreach ($this->getDefinitions() as $plugin_id => $plugin_definition) { + // Add the plugin to the list. + $options[$plugin_id] = String::checkPlain($plugin_definition['label']); + } + return $options; + } + +} diff --git a/search_api.module b/search_api.module index cba6387..7ed66af 100644 --- a/search_api.module +++ b/search_api.module @@ -56,9 +56,7 @@ function search_api_entity_insert(EntityInterface $entity) { $indexes = $storage->getIndexesForEntity($entity); foreach ($indexes as $index) { - if ($index->status()) { - $index->getDatasource()->trackInsert(array($entity->id())); - } + $index->insertItems($index->getDatasource(), array($entity->id())); } } } @@ -77,9 +75,7 @@ function search_api_entity_update(EntityInterface $entity) { $indexes = $storage->getIndexesForEntity($entity); foreach ($indexes as $index) { - if ($index->status()) { - $index->getDatasource()->trackUpdate(array($entity->id())); - } + $index->updateItems($index->getDatasource(), array($entity->id())); } } } @@ -98,9 +94,7 @@ function search_api_entity_delete(EntityInterface $entity) { $indexes = $storage->getIndexesForEntity($entity); foreach ($indexes as $index) { - if ($index->status()) { - $index->getDatasource()->trackDelete(array($entity->id())); - } + $index->deleteItems($index->getDatasource(), array($entity->id())); } } } diff --git a/search_api.services.yml b/search_api.services.yml index 0b67dea..b860a81 100644 --- a/search_api.services.yml +++ b/search_api.services.yml @@ -10,3 +10,7 @@ services: search_api.datasource.plugin.manager: class: Drupal\search_api\Datasource\DatasourcePluginManager parent: default_plugin_manager + + plugin.manager.search_api.tracker: + class: Drupal\search_api\Tracker\TrackerPluginManager + parent: default_plugin_manager diff --git a/search_api.theme.inc b/search_api.theme.inc index 0192f49..cc44573 100644 --- a/search_api.theme.inc +++ b/search_api.theme.inc @@ -218,6 +218,8 @@ function theme_search_api_index($variables) { $server = $index->hasValidServer() ? $index->getServer() : NULL; /** @var $server \Drupal\search_api\DataSource\DataSourceInterface */ $datasource = $index->hasValidDatasource() ? $index->getDatasource() : NULL; + /** @var $tracker \Drupal\search_api\Tracker\TrackerInterface */ + $tracker = $index->hasValidTracker() ? $index->getTracker() : NULL; // Initialize the output variable to an empty string. $output = ''; @@ -282,6 +284,22 @@ function theme_search_api_index($variables) { $rows[] = Utility::deepCopy($row); $class = ''; + // Check if a tracker is available. + if ($tracker) { + // Get the tracker plugin definition. + $tracker_plugin_definition = $tracker->getPluginDefinition(); + // Use the label defined in the plugin definition. + $info = $tracker_plugin_definition['label']; + } + else { + $class = 'error'; + $info = t('Invalid or missing tracker plugin'); + } + // Append the row and reset variables. + $label = t('Tracker'); + $rows[] = Utility::deepCopy($row); + $class = ''; + // Check if a server is available. if ($server) { $label = t('Server'); @@ -323,11 +341,11 @@ function theme_search_api_index($variables) { $label = t('Cron batch size'); $rows[] = Utility::deepCopy($row); $class = ''; - // Check if a valid datasource is available. - if ($datasource) { + // Check if a valid tracker is available. + if ($tracker) { // Get the indexed and total item count. - $indexed_count = $datasource->getIndexedItemsCount(); - $total_count = $datasource->getTotalItemsCount(); + $indexed_count = $tracker->getIndexedItemsCount(); + $total_count = $tracker->getTotalItemsCount(); // Calculate the index progress. $percent = ($total_count !== 0) ? (100 * $indexed_count / $total_count) : 0;