diff --git a/.travis.yml b/.travis.yml index b11b1e5..80fff31 100644 --- a/.travis.yml +++ b/.travis.yml @@ -63,7 +63,9 @@ install: - composer require drupal/search_api:1.x-dev - composer require drupal/search_api_autocomplete:1.x-dev - composer require drupal/search_api_solr:999.0.0 + - composer require drupal/search_api_location:1.x-dev - composer require drupal/facets:1.x-dev + - composer require drupal/geofield:1.x-dev - composer require drush/drush # Patch template. ######################################### diff --git a/src/Plugin/search_api/backend/SearchApiSolrBackend.php b/src/Plugin/search_api/backend/SearchApiSolrBackend.php index bb64af5..d0593c9 100644 --- a/src/Plugin/search_api/backend/SearchApiSolrBackend.php +++ b/src/Plugin/search_api/backend/SearchApiSolrBackend.php @@ -16,7 +16,9 @@ use Drupal\Core\Language\LanguageInterface; use Drupal\Core\Language\LanguageManagerInterface; use Drupal\Core\Plugin\PluginFormInterface; use Drupal\Core\TypedData\ComplexDataDefinitionInterface; +use Drupal\Core\TypedData\DataDefinition; use Drupal\Core\Url; +use Drupal\search_api\Item\Field; use Drupal\search_api\Item\FieldInterface; use Drupal\search_api\Item\ItemInterface; use Drupal\search_api\Plugin\PluginFormTrait; @@ -458,9 +460,9 @@ class SearchApiSolrBackend extends BackendPluginBase implements SolrBackendInter 'search_api_facets_operator_or', 'search_api_mlt', 'search_api_random_sort', + 'search_api_data_type_location', // 'search_api_grouping', // 'search_api_spellcheck', - // 'search_api_data_type_location', // 'search_api_data_type_geohash', ]; } @@ -470,6 +472,7 @@ class SearchApiSolrBackend extends BackendPluginBase implements SolrBackendInter */ public function supportsDataType($type) { return in_array($type, [ + 'location', 'solr_text_ngram', 'solr_text_phonetic', 'solr_text_unstemmed', @@ -944,7 +947,7 @@ class SearchApiSolrBackend extends BackendPluginBase implements SolrBackendInter // Set sorts. $this->setSorts($solarium_query, $query, $field_names); - // Set facet fields. + // Set facet fields. setSpatial() might add more facets. $this->setFacets($query, $solarium_query, $field_names); // Handle spatial filters. @@ -976,19 +979,17 @@ class SearchApiSolrBackend extends BackendPluginBase implements SolrBackendInter } } - /** - * @todo Make this more configurable so that views can choose which fields - * it wants to fetch - */ + // @todo Make this more configurable so that views can choose which fields + // it wants to fetch. if (!empty($this->configuration['retrieve_data'])) { - $solarium_query->setFields(['*', 'score']); + $solarium_query->addFields(['*', 'score']); } else { $returned_fields = [SEARCH_API_ID_FIELD_NAME, 'score']; if (!$this->configuration['site_hash']) { $returned_fields[] = 'hash'; } - $solarium_query->setFields($returned_fields); + $solarium_query->addFields($returned_fields); } $this->applySearchWorkarounds($solarium_query, $query); @@ -1128,6 +1129,14 @@ class SearchApiSolrBackend extends BackendPluginBase implements SolrBackendInter } $name = $pref . '_' . $key; $ret[$key] = SearchApiSolrUtility::encodeSolrName($name); + + // Add a distance pseudo field for any location field. These fields + // don't really exist in the solr core, but we tell solr to name the + // distance calculation results that way. Later we directly pass these + // as "fields" to Drupal and especially Views. + if ($type == 'location') { + $ret[$key . '__distance'] = SearchApiSolrUtility::encodeSolrName($name . '__distance'); + } } } @@ -1341,7 +1350,7 @@ class SearchApiSolrBackend extends BackendPluginBase implements SolrBackendInter $index = $query->getIndex(); $backend_config = $index->getServerInstance()->getBackendConfig(); $field_names = $this->getSolrFieldNames($index); - $fields = $index->getFields(); + $fields = $index->getFields(TRUE); $site_hash = SearchApiSolrUtility::getSiteHash(); // We can find the item ID and the score in the special 'search_api_*' // properties. Mappings are provided for these properties in @@ -1529,16 +1538,17 @@ class SearchApiSolrBackend extends BackendPluginBase implements SolrBackendInter if (isset($result_data['facet_counts']['facet_queries'])) { if ($spatials = $query->getOption('search_api_location')) { foreach ($result_data['facet_counts']['facet_queries'] as $key => $count) { - if (!preg_match('/^spatial-(.*)-(\d+(?:\.\d+)?)$/', $key, $m)) { + // This special key is defined in setSpatial(). + if (!preg_match('/^spatial-(.*)-(\d+(?:\.\d+)?)-(\d+(?:\.\d+)?)$/', $key, $matches)) { continue; } - if (empty($extract_facets[$m[1]])) { + if (empty($extract_facets[$matches[1]])) { continue; } - $facet = $extract_facets[$m[1]]; + $facet = $extract_facets[$matches[1]]; if ($count >= $facet['min_count']) { - $facets[$m[1]][] = array( - 'filter' => "[* {$m[2]}]", + $facets[$matches[1]][] = array( + 'filter' => "[{$matches[2]} {$matches[3]}]", 'count' => $count, ); } @@ -2110,6 +2120,7 @@ class SearchApiSolrBackend extends BackendPluginBase implements SolrBackendInter if (!empty($this->configuration['highlight_data'])) { $item_fields = $item->getFields(); foreach ($field_mapping as $search_api_property => $solr_property) { + // @todo We now have more fulltext field variants to be highlighted. if ((strpos($solr_property, 'ts_') === 0 || strpos($solr_property, 'tm_') === 0) && !empty($data['highlighting'][$solr_id][$solr_property])) { $snippets = []; foreach ($data['highlighting'][$solr_id][$solr_property] as $value) { @@ -2364,28 +2375,32 @@ class SearchApiSolrBackend extends BackendPluginBase implements SolrBackendInter /** * Adds spatial features to the search query. * - * @todo This code is outdated and needs to be reviewed and refactored. - * * @param \Solarium\QueryType\Select\Query\Query $solarium_query * The solr query. * @param \Drupal\search_api\Query\QueryInterface $query * The search api query. * @param array $spatial_options * The spatial options to add. - * @param $field_names + * @param array $field_names * The field names, to add the spatial options for. */ protected function setSpatial(Query $solarium_query, QueryInterface $query, $spatial_options = array(), $field_names = array()) { + $helper = $solarium_query->getHelper(); + foreach ($spatial_options as $i => $spatial) { // Reset radius for each option. unset($radius); + unset($min_radius); if (empty($spatial['field']) || empty($spatial['lat']) || empty($spatial['lon'])) { continue; } - $field = $field_names[$spatial['field']]; - $point = ((float) $spatial['lat']) . ',' . ((float) $spatial['lon']); + $solr_field = $field_names[$spatial['field']]; + $distance_field = $spatial['field'] . '__distance'; + $solr_distance_field = $field_names[$distance_field]; + $spatial['lat'] = (float) $spatial['lat']; + $spatial['lon'] = (float) $spatial['lon']; // Prepare the filter settings. if (isset($spatial['radius'])) { @@ -2402,8 +2417,8 @@ class SearchApiSolrBackend extends BackendPluginBase implements SolrBackendInter foreach ($filter_queries as $key => $filter_query) { // If the fq consists only of a filter on this field, replace it with // a range. - $preg_field = preg_quote($field, '/'); - if (preg_match('/^' . $preg_field . ':\["?(\*|\d+(?:\.\d+)?)"? TO "?(\*|\d+(?:\.\d+)?)"?\]$/', $filter_query, $matches)) { + $preg_field = preg_quote($solr_field, '/'); + if (preg_match('/^' . $preg_field . ':\["?(\*|\d+(?:\.\d+)?)"? TO "?(\*|\d+(?:\.\d+)?)"?\]$/', $filter_query->getQuery(), $matches)) { unset($filter_queries[$key]); if ($matches[1] && is_numeric($matches[1])) { $min_radius = isset($min_radius) ? max($min_radius, $matches[1]) : $matches[1]; @@ -2414,58 +2429,70 @@ class SearchApiSolrBackend extends BackendPluginBase implements SolrBackendInter } } } + $solarium_query->clearFilterQueries(); + $solarium_query->addFilterQueries($filter_queries); + + $geodist = $helper->geodist($solr_field, $spatial['lat'], $spatial['lon']); // If either a radius was given in the option, or a filter was // encountered, set a filter for the lowest value. If a lower boundary // was set (too), we can only set a filter for that if the field name // doesn't contains any colons. - if (isset($min_radius) && strpos($field, ':') === FALSE) { + if (isset($min_radius)) { $upper = isset($radius) ? " u=$radius" : ''; - $solarium_query->createFilterQuery($field)->setQuery("{!frange l=$min_radius$upper}geodist($field,$point)"); + $solarium_query->createFilterQuery($solr_field)->setQuery("{!frange l=$min_radius$upper}" . $geodist); } elseif (isset($radius)) { - $solarium_query->createFilterQuery($field)->setQuery("{!$spatial_method pt=$point sfield=$field d=$radius}"); + $solarium_query->createFilterQuery($solr_field)->setQuery($helper->{$spatial_method}($solr_field, $spatial['lat'], $spatial['lon'], $radius)); } - // @todo: Check if this object returns the correct value + $solarium_query->addField($solr_distance_field . ':' . $geodist); + $sorts = $solarium_query->getSorts(); // Change sort on the field, if set (and not already changed). - if (isset($sorts[$spatial['field']]) && substr($sorts[$spatial['field']], 0, strlen($field)) === $field) { - $sorts[$spatial['field']] = str_replace($field, "geodist($field,$point)", $sorts[$spatial['field']]); + if (isset($sorts[$solr_distance_field])) { + $solarium_query->setQuery('{!func}' . $geodist); + $sorts['score'] = $sorts[$solr_distance_field]; + unset($sorts[$solr_distance_field]); + $solarium_query->clearSorts(); + $solarium_query->addSorts($sorts); } // Change the facet parameters for spatial fields to return distance // facets. - $facets = $solarium_query->getFacetSet(); - // @todo: Fix this so it takes it from the solarium query - if (!empty($facets)) { - if (!empty($facet_params['facet.field'])) { - $facet_params['facet.field'] = array_diff($facet_params['facet.field'], array($field)); - } + $facet_set = $solarium_query->getFacetSet(); + if (!empty($facet_set)) { + /** @var \Solarium\QueryType\Select\Query\Component\Facet\Field[] $facets */ + $facets = $facet_set->getFacets(); foreach ($facets as $delta => $facet) { - if ($facet['field'] != $spatial['field']) { + $facet_options = $facet->getOptions(); + if ($facet_options['field'] != $solr_distance_field) { continue; } - $steps = $facet['limit'] > 0 ? $facet['limit'] : 5; - $step = (isset($radius) ? $radius : 100) / $steps; - for ($k = $steps - 1; $k > 0; --$k) { - $distance = $step * $k; - $key = "spatial-$delta-$distance"; - $facet_params['facet.query'][] = "{!$spatial_method pt=$point sfield=$field d=$distance key=$key}"; - } - foreach (array('limit', 'mincount', 'missing') as $setting) { - unset($facet_params["f.$field.facet.$setting"]); + $facet_set->removeFacet($delta); + + $limit = $facet->getLimit(); + + // @todo : Check if these defaults make any sense. + $steps = $limit > 0 ? $limit : 5; + $min = isset($min_radius) ? $min_radius : 0; + $max = isset($radius) ? $radius : 100; + $step = ($max - $min) / $steps; + + for ($distance_min = $min; $distance_min <= $max - $step; $distance_min += $step) { + $distance_max = $distance_min + $step - 1; + // Define our own facet key to transport the min and max values. + // These will be extracted in extractFacets(). + $key = "spatial-{$distance_field}-{$distance_min}-{$distance_max}"; + // Due to a limitation/bug in Solarium, it is not possible to use + // setQuery method for geo facets. + // So the key is misused to get a correct query. + // @see https://github.com/solariumphp/solarium/issues/229 + $facet_set->createFacetQuery($key . ' frange l=' . $distance_min . ' u=' . $distance_max)->setQuery($geodist); } } } } - - // Normal sorting on location fields isn't possible. - foreach (array_keys($solarium_query->getSorts()) as $sort) { - if (substr($sort, 0, 3) === 'loc') { - $solarium_query->removeSort($sort); - } - } } /** @@ -2588,4 +2615,27 @@ class SearchApiSolrBackend extends BackendPluginBase implements SolrBackendInter return $connector->getContentFromExtractResult($result, $filepath); } + /** + * {@inheritdoc} + */ + public function getBackendDefinedFields(IndexInterface $index) { + $location_distance_fields = []; + + foreach ($index->getFields() as $field) { + if ($field->getType() == 'location') { + $distance_field_name = $field->getFieldIdentifier() . '__distance'; + $distance_field = new Field($index, $distance_field_name); + $distance_field->setLabel($field->getLabel() . ' (distance)'); + $distance_field->setDataDefinition(DataDefinition::create('decimal')); + $distance_field->setType('decimal'); + $distance_field->setDatasourceId($field->getDatasourceId()); + $distance_field->setPropertyPath($distance_field_name); + + $location_distance_fields[$distance_field_name] = $distance_field; + } + } + + return $location_distance_fields; + } + } diff --git a/tests/src/Kernel/SearchApiSolrLocationTest.php b/tests/src/Kernel/SearchApiSolrLocationTest.php new file mode 100644 index 0000000..cf9965a --- /dev/null +++ b/tests/src/Kernel/SearchApiSolrLocationTest.php @@ -0,0 +1,329 @@ +installConfig([ + 'search_api_solr', + 'search_api_solr_test', + ]); + $this->installEntitySchema('field_storage_config'); + $this->installEntitySchema('field_config'); + + // Create a location field and storage for testing. + FieldStorageConfig::create([ + 'field_name' => 'location', + 'entity_type' => 'entity_test_mulrev_changed', + 'type' => 'geofield', + ])->save(); + FieldConfig::create([ + 'entity_type' => 'entity_test_mulrev_changed', + 'field_name' => 'location', + 'bundle' => 'item', + ])->save(); + + $this->insertExampleContent(); + + /** @var \Drupal\search_api\Entity\Index $index */ + $index = Index::load($this->indexId); + + $info = [ + 'datasource_id' => 'entity:entity_test_mulrev_changed', + 'property_path' => 'location', + 'type' => 'location', + ]; + + $fieldsHelper = $this->container->get('search_api.fields_helper'); + + $index->addField($fieldsHelper->createField($index, 'location', $info)); + $index->save(); + + /** @var \Drupal\search_api\Entity\Server $server */ + $server = Server::load($this->serverId); + + $config = $server->getBackendConfig(); + $config['retrieve_data'] = TRUE; + $server->setBackendConfig($config); + $server->save(); + + $this->detectSolrAvailability(); + + $this->indexItems($this->indexId); + } + + /** + * {@inheritdoc} + */ + protected function indexItems($index_id) { + $index_status = parent::indexItems($index_id); + sleep($this->waitForCommit); + return $index_status; + } + + /** + * Detects the availability of a Solr Server and sets $this->solrAvailable. + */ + protected function detectSolrAvailability() { + // Because this is a kernel test, the routing isn't built by default, so + // we have to force it. + \Drupal::service('router.builder')->rebuild(); + + try { + $backend = Server::load($this->serverId)->getBackend(); + if ($backend->isAvailable()) { + $this->solrAvailable = TRUE; + } + } + catch (\Exception $e) { + } + } + + /** + * {@inheritdoc} + */ + public function insertExampleContent() { + $this->addTestEntity(1, [ + 'name' => 'London', + 'body' => 'London', + 'type' => 'item', + 'location' => 'POINT(-0.076132 51.508530)', + ]); + $this->addTestEntity(2, [ + 'name' => 'New York', + 'body' => 'New York', + 'type' => 'item', + 'location' => 'POINT(-73.138260 40.792240)', + ]); + $this->addTestEntity(3, [ + 'name' => 'Brussels', + 'body' => 'Brussels', + 'type' => 'item', + 'location' => 'POINT(4.355607 50.878899)', + ]); + $count = \Drupal::entityQuery('entity_test_mulrev_changed')->count()->execute(); + $this->assertEquals(3, $count, "$count items inserted."); + } + + /** + * Tests highlight and excerpt options. + */ + public function testBackend() { + + // Search 500km from Antwerp. + $location_options = [ + [ + 'field' => 'location', + 'lat' => '51.260197', + 'lon' => '4.402771', + 'radius' => '500', + ], + ]; + /** @var \Drupal\search_api\Query\ResultSet $result */ + $query = $this->buildSearch(NULL, [], NULL, FALSE) + ->sort('location__distance'); + + $query->setOption('search_api_location', $location_options); + $result = $query->execute(); + + $this->assertResults([3, 1], $result, 'Search for 500km from Antwerp ordered by distance'); + + /** @var \Drupal\search_api\Item\Item $item */ + $item = $result->getResultItems()['entity:entity_test_mulrev_changed/3:en']; + $distance = $item->getField('location__distance')->getValues()[0]; + + $this->assertEquals(42.5263374675, $distance, 'The distance is correctly returned'); + + // Search between 100km and 6000km from Antwerp. + $location_options = [ + [ + 'field' => 'location', + 'lat' => '51.260197', + 'lon' => '4.402771', + ], + ]; + $query = $this->buildSearch(NULL, [], NULL, FALSE) + ->addCondition('location', ['100', '6000'], 'BETWEEN') + ->sort('location__distance', 'DESC'); + + $query->setOption('search_api_location', $location_options); + $result = $query->execute(); + + $this->assertResults([2, 1], $result, 'Search between 100 and 6000km from Antwerp ordered by distance descending'); + + $facets_options['location__distance'] = [ + 'field' => 'location__distance', + 'limit' => 10, + 'min_count' => 0, + 'missing' => TRUE, + ]; + + // Search 1000km from Antwerp. + $location_options = [ + [ + 'field' => 'location', + 'lat' => '51.260197', + 'lon' => '4.402771', + 'radius' => '1000', + ], + ]; + $query = $this->buildSearch(NULL, [], NULL, FALSE) + ->sort('location__distance'); + + $query->setOption('search_api_location', $location_options); + $query->setOption('search_api_facets', $facets_options); + $result = $query->execute(); + $facets = $result->getExtraData('search_api_facets', [])['location__distance']; + + $expected = [ + [ + 'filter' => '[0 199]', + 'count' => 1, + ], + [ + 'filter' => '[200 399]', + 'count' => 1, + ], + [ + 'filter' => '[400 599]', + 'count' => 0, + ], + [ + 'filter' => '[600 799]', + 'count' => 0, + ], + [ + 'filter' => '[800 999]', + 'count' => 0, + ], + ]; + + $this->assertEquals($expected, $facets, 'The correct location facets are returned'); + + $facets_options['location__distance'] = [ + 'field' => 'location__distance', + 'limit' => 3, + 'min_count' => 1, + 'missing' => TRUE, + ]; + + // Search between 100km and 1000km from Antwerp. + $location_options = [ + [ + 'field' => 'location', + 'lat' => '51.260197', + 'lon' => '4.402771', + 'radius' => '1000', + ], + ]; + + $query = $this->buildSearch(NULL, [], NULL, FALSE) + ->addCondition('location',['100', '1000'], 'BETWEEN') + ->sort('location__distance'); + + $query->setOption('search_api_location', $location_options); + $query->setOption('search_api_facets', $facets_options); + $result = $query->execute(); + + $facets = $result->getExtraData('search_api_facets', [])['location__distance']; + + $expected = [ + [ + 'filter' => '[100 399]', + 'count' => 1, + ], + ]; + + $this->assertEquals($expected, $facets, 'The correct location facets are returned'); + } + + /** + * {@inheritdoc} + */ + protected function checkServerBackend() {} + + /** + * {@inheritdoc} + */ + protected function updateIndex() {} + + /** + * {@inheritdoc} + */ + protected function checkSecondServer() {} + + /** + * {@inheritdoc} + */ + protected function checkModuleUninstall() {} + +}