diff --git a/.travis.yml b/.travis.yml index b11b1e5..9e3cd41 100644 --- a/.travis.yml +++ b/.travis.yml @@ -64,6 +64,7 @@ install: - composer require drupal/search_api_autocomplete:1.x-dev - composer require drupal/search_api_solr:999.0.0 - composer require drupal/facets:1.x-dev + - composer require drupal/geofield:1.x-dev - composer require drush/drush # Patch template. ######################################### diff --git a/.travis.yml.orig b/.travis.yml.orig new file mode 100644 index 0000000..b11b1e5 --- /dev/null +++ b/.travis.yml.orig @@ -0,0 +1,96 @@ +language: php +cache: + bundler: true + directories: + - $HOME/tmp/drush + - $HOME/.bundle + - $HOME/.composer + - $HOME/downloads + apt: true + +git: + depth: 10000 + +php: + - 7.0 + - 5.5 + +env: + # 4.5.1 is the oldest version we support + - PATH=$PATH:/home/travis/.composer/vendor/bin SOLR_VERSION=4.5.1 SOLR_CORE=d8 SOLR_CONFS="$TRAVIS_BUILD_DIR/solr-conf/4.x" + # 5.5.0 introduced major changes for boolean operators. + - PATH=$PATH:/home/travis/.composer/vendor/bin SOLR_VERSION=5.5.4 SOLR_CORE=d8 SOLR_CONFS="$TRAVIS_BUILD_DIR/solr-conf/5.x" + # 6.4.2 is the latest currently supported release. + - PATH=$PATH:/home/travis/.composer/vendor/bin SOLR_VERSION=6.4.2 SOLR_CORE=d8 SOLR_CONFS="$TRAVIS_BUILD_DIR/solr-conf/6.x" + +notifications: + irc: + - "chat.freenode.net#drupal-search-api" + +# This will create the database +mysql: + database: drupal + username: root + encoding: utf8 + +# To be able to run a webbrowser +# If we need anything more powerful +# than e.g. phantomjs +before_install: + - phpenv config-rm xdebug.ini + - composer self-update + - composer global require "hirak/prestissimo:^0.3" + - sudo apt-get update -qq > /dev/null + - "export DISPLAY=:99.0" + - "sh -e /etc/init.d/xvfb start" + +install: + - git tag 999.0.0 + # Make sure we don't fail when checking out projects + - echo -e "Host github.com\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config + - echo -e "Host git.drupal.org\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config + # Set sendmail so drush doesn't throw an error during site install. + - echo "sendmail_path='true'" >> `php --ini | grep "Loaded Configuration" | awk '{print $4}'` + # Forward the errors to the syslog so we can print them + - echo "error_log=syslog" >> `php --ini | grep "Loaded Configuration" | awk '{print $4}'` + # Get latest Drupal 8 core + - cd $TRAVIS_BUILD_DIR/.. + - git clone --depth=1 --branch 8.3.x https://git.drupal.org/project/drupal.git + - cd $TRAVIS_BUILD_DIR/../drupal + - composer install + - composer config repositories.drupal composer https://packages.drupal.org/8 + - composer config repositories.search_api_solr vcs $TRAVIS_BUILD_DIR + - composer require drupal/search_api:1.x-dev + - composer require drupal/search_api_autocomplete:1.x-dev + - composer require drupal/search_api_solr:999.0.0 + - composer require drupal/facets:1.x-dev + - composer require drush/drush + # Patch template. + ######################################### + # to be removed once #2824932 is resolved + #- cd modules/search_api + #- curl https://www.drupal.org/files/issues/2824932.patch | patch -p1 + #- cd $TRAVIS_BUILD_DIR/../drupal + ######################################### + +before_script: + # Start the built-in php web server (mysql is already started) and + # suppress web-server access logs output. + - php -S localhost:8888 >& /dev/null & + # Install the site + - ./vendor/bin/drush -v site-install minimal --db-url=mysql://root:@localhost/drupal --yes + - ./vendor/bin/drush en --yes simpletest + # Install Solr + - cat $TRAVIS_BUILD_DIR/travis-solr.sh | bash + +script: + # Run the tests + - cd $TRAVIS_BUILD_DIR/../drupal + - export SIMPLETEST_DB=mysql://root:@localhost/drupal + - export SIMPLETEST_BASE_URL=http://localhost:8888 + - ./vendor/bin/phpunit -c core --group search_api_solr --verbose --debug | tee ; export TEST_PHPUNIT=${PIPESTATUS[0]} ; echo $TEST_PHPUNIT + # Re-enable when trying to get CodeSniffer doesn't return a 403 anymore. + #- /home/travis/.composer/vendor/bin/phpcs --standard=/home/travis/.composer/vendor/drupal/coder/coder_sniffer/Drupal --extensions=php,inc,test,module,install --ignore=css/ $TRAVIS_BUILD_DIR/../drupal/modules/search_api + # Exit the build + - echo $TEST_PHPUNIT + - if [ $TEST_PHPUNIT -eq 0 ]; then exit 0; else exit 1; fi diff --git a/src/Plugin/search_api/backend/SearchApiSolrBackend.php b/src/Plugin/search_api/backend/SearchApiSolrBackend.php index 624dd7b..31c421c 100644 --- a/src/Plugin/search_api/backend/SearchApiSolrBackend.php +++ b/src/Plugin/search_api/backend/SearchApiSolrBackend.php @@ -15,6 +15,7 @@ use Drupal\Core\Language\LanguageManagerInterface; use Drupal\Core\Plugin\PluginFormInterface; use Drupal\Core\TypedData\ComplexDataDefinitionInterface; use Drupal\Core\Url; +use Drupal\search_api\Item\Field; use Drupal\search_api\Item\FieldInterface; use Drupal\search_api\Item\ItemInterface; use Drupal\search_api\Plugin\PluginFormTrait; @@ -45,6 +46,7 @@ use Solarium\QueryType\Suggester\Query as SuggesterQuery; use Solarium\QueryType\Suggester\Result\Result as SuggesterResult; use Solarium\QueryType\Update\Query\Document\Document; use Symfony\Component\DependencyInjection\ContainerInterface; +use Drupal\Core\TypedData\DataDefinition; /** * The minimum required Solr schema version. @@ -466,7 +468,7 @@ class SearchApiSolrBackend extends BackendPluginBase implements SolrBackendInter // 'search_api_mlt', 'search_api_random_sort', // 'search_api_spellcheck', - // 'search_api_data_type_location', + 'search_api_data_type_location', // 'search_api_data_type_geohash', ); } @@ -872,6 +874,8 @@ class SearchApiSolrBackend extends BackendPluginBase implements SolrBackendInter $connector = $this->getSolrConnector(); // Instantiate a Solarium select query. $solarium_query = $connector->getSelectQuery(); + // Clear all the fields so we start with a clean slate. + $solarium_query->clearFields(); $query_helper = $connector->getQueryHelper($solarium_query); // Extract keys. @@ -968,19 +972,16 @@ class SearchApiSolrBackend extends BackendPluginBase implements SolrBackendInter } } - /** - * @todo Make this more configurable so that views can choose which fields - * it wants to fetch - */ + // @todo Make this more configurable so that views can choose which fields it wants to fetch if (!empty($this->configuration['retrieve_data'])) { - $solarium_query->setFields(['*', 'score']); + $solarium_query->addFields(['*', 'score']); } else { $returned_fields = [SEARCH_API_ID_FIELD_NAME, 'score']; if (!$this->configuration['site_hash']) { $returned_fields[] = 'hash'; } - $solarium_query->setFields($returned_fields); + $solarium_query->addFields($returned_fields); } $this->applySearchWorkarounds($solarium_query, $query); @@ -1117,6 +1118,11 @@ class SearchApiSolrBackend extends BackendPluginBase implements SolrBackendInter } $name = $pref . '_' . $key; $ret[$key] = SearchApiSolrUtility::encodeSolrName($name); + + // Add the distance pseudo field for location fields. + if ($type == 'location') { + $ret[$key . '__distance'] = SearchApiSolrUtility::encodeSolrName($name . '__distance'); + } } } @@ -1330,7 +1336,7 @@ class SearchApiSolrBackend extends BackendPluginBase implements SolrBackendInter $index = $query->getIndex(); $backend_config = $index->getServerInstance()->getBackendConfig(); $field_names = $this->getSolrFieldNames($index); - $fields = $index->getFields(); + $fields = $index->getFields(TRUE); $site_hash = SearchApiSolrUtility::getSiteHash(); // We can find the item ID and the score in the special 'search_api_*' // properties. Mappings are provided for these properties in @@ -1518,7 +1524,7 @@ class SearchApiSolrBackend extends BackendPluginBase implements SolrBackendInter if (isset($result_data['facet_counts']['facet_queries'])) { if ($spatials = $query->getOption('search_api_location')) { foreach ($result_data['facet_counts']['facet_queries'] as $key => $count) { - if (!preg_match('/^spatial-(.*)-(\d+(?:\.\d+)?)$/', $key, $m)) { + if (!preg_match('/^spatial-(.*)-(\d+(?:\.\d+)?)-(\d+(?:\.\d+)?)$/', $key, $m)) { continue; } if (empty($extract_facets[$m[1]])) { @@ -1527,7 +1533,7 @@ class SearchApiSolrBackend extends BackendPluginBase implements SolrBackendInter $facet = $extract_facets[$m[1]]; if ($count >= $facet['min_count']) { $facets[$m[1]][] = array( - 'filter' => "[* {$m[2]}]", + 'filter' => "[{$m[2]} {$m[3]}]", 'count' => $count, ); } @@ -2308,20 +2314,25 @@ class SearchApiSolrBackend extends BackendPluginBase implements SolrBackendInter * The search api query. * @param array $spatial_options * The spatial options to add. - * @param $field_names + * @param array $field_names * The field names, to add the spatial options for. */ protected function setSpatial(Query $solarium_query, QueryInterface $query, $spatial_options = array(), $field_names = array()) { + $helper = $solarium_query->getHelper(); + foreach ($spatial_options as $i => $spatial) { // Reset radius for each option. unset($radius); + unset($min_radius); if (empty($spatial['field']) || empty($spatial['lat']) || empty($spatial['lon'])) { continue; } $field = $field_names[$spatial['field']]; - $point = ((float) $spatial['lat']) . ',' . ((float) $spatial['lon']); + $distance_field = $field . '__distance'; + $spatial['lat'] = (float) $spatial['lat']; + $spatial['lon'] = (float) $spatial['lon']; // Prepare the filter settings. if (isset($spatial['radius'])) { @@ -2339,7 +2350,7 @@ class SearchApiSolrBackend extends BackendPluginBase implements SolrBackendInter // If the fq consists only of a filter on this field, replace it with // a range. $preg_field = preg_quote($field, '/'); - if (preg_match('/^' . $preg_field . ':\["?(\*|\d+(?:\.\d+)?)"? TO "?(\*|\d+(?:\.\d+)?)"?\]$/', $filter_query, $matches)) { + if (preg_match('/^' . $preg_field . ':\["?(\*|\d+(?:\.\d+)?)"? TO "?(\*|\d+(?:\.\d+)?)"?\]$/', $filter_query->getQuery(), $matches)) { unset($filter_queries[$key]); if ($matches[1] && is_numeric($matches[1])) { $min_radius = isset($min_radius) ? max($min_radius, $matches[1]) : $matches[1]; @@ -2350,58 +2361,69 @@ class SearchApiSolrBackend extends BackendPluginBase implements SolrBackendInter } } } + $solarium_query->clearFilterQueries(); + $solarium_query->addFilterQueries($filter_queries); + + $geodist = $helper->geodist($field, $spatial['lat'], $spatial['lon']); // If either a radius was given in the option, or a filter was // encountered, set a filter for the lowest value. If a lower boundary // was set (too), we can only set a filter for that if the field name // doesn't contains any colons. - if (isset($min_radius) && strpos($field, ':') === FALSE) { + if (isset($min_radius)) { $upper = isset($radius) ? " u=$radius" : ''; - $solarium_query->createFilterQuery($field)->setQuery("{!frange l=$min_radius$upper}geodist($field,$point)"); + $solarium_query->createFilterQuery($field)->setQuery("{!frange l=$min_radius$upper}" . $geodist); } elseif (isset($radius)) { - $solarium_query->createFilterQuery($field)->setQuery("{!$spatial_method pt=$point sfield=$field d=$radius}"); + $solarium_query->createFilterQuery($field)->setQuery($helper->{$spatial_method}($field, $spatial['lat'], $spatial['lon'], $radius)); } - // @todo: Check if this object returns the correct value + $solarium_query->addField($distance_field . ':' . $geodist); + $sorts = $solarium_query->getSorts(); // Change sort on the field, if set (and not already changed). - if (isset($sorts[$spatial['field']]) && substr($sorts[$spatial['field']], 0, strlen($field)) === $field) { - $sorts[$spatial['field']] = str_replace($field, "geodist($field,$point)", $sorts[$spatial['field']]); + if (isset($sorts[$distance_field])) { + $solarium_query->setQuery('{!func}' . $geodist); + $sorts['score'] = $sorts[$distance_field]; + unset($sorts[$distance_field]); + $solarium_query->clearSorts(); + $solarium_query->addSorts($sorts); } // Change the facet parameters for spatial fields to return distance // facets. - $facets = $solarium_query->getFacetSet(); - // @todo: Fix this so it takes it from the solarium query - if (!empty($facets)) { - if (!empty($facet_params['facet.field'])) { - $facet_params['facet.field'] = array_diff($facet_params['facet.field'], array($field)); - } + $facet_set = $solarium_query->getFacetSet(); + if (!empty($facet_set)) { + /** @var \Solarium\QueryType\Select\Query\Component\Facet\Field[] $facets */ + $facets = $facet_set->getFacets(); foreach ($facets as $delta => $facet) { - if ($facet['field'] != $spatial['field']) { + $facet_options = $facet->getOptions(); + if ($facet_options['field'] != $distance_field) { continue; } - $steps = $facet['limit'] > 0 ? $facet['limit'] : 5; - $step = (isset($radius) ? $radius : 100) / $steps; - for ($k = $steps - 1; $k > 0; --$k) { - $distance = $step * $k; - $key = "spatial-$delta-$distance"; - $facet_params['facet.query'][] = "{!$spatial_method pt=$point sfield=$field d=$distance key=$key}"; - } - foreach (array('limit', 'mincount', 'missing') as $setting) { - unset($facet_params["f.$field.facet.$setting"]); + $facet_set->removeFacet($delta); + + $limit = $facet->getLimit(); + + // @todo : Check if these defaults make any sense. + $steps = $limit > 0 ? $limit : 5; + $min = isset($min_radius) ? $min_radius : 0; + $max = isset($radius) ? $radius : 100; + $step = ($max - $min) / $steps; + + for ($distance_min = $min; $distance_min <= $max - $step; $distance_min += $step) { + $distance_max = $distance_min + $step - 1; + $key = "spatial-{$spatial['field']}__distance-{$distance_min}-{$distance_max}"; + + // Due to a limitation/bug in Solarium, it is not possible to use + // setQuery method for geo facets. + // So the key is misused to get a correct query. + // @see https://github.com/solariumphp/solarium/issues/229 + $facet_set->createFacetQuery($key . ' frange l=' . $distance_min . ' u=' . $distance_max)->setQuery($geodist); } } } } - - // Normal sorting on location fields isn't possible. - foreach (array_keys($solarium_query->getSorts()) as $sort) { - if (substr($sort, 0, 3) === 'loc') { - $solarium_query->removeSort($sort); - } - } } /** @@ -2524,4 +2546,37 @@ class SearchApiSolrBackend extends BackendPluginBase implements SolrBackendInter return $connector->getContentFromExtractResult($result, $filepath); } + /** + * {@inheritdoc} + */ + public function supportsDataType($type) { + $supported_data_types = [ + 'location', + ]; + return in_array($type, $supported_data_types); + } + + /** + * {@inheritdoc} + */ + public function getBackendDefinedFields(IndexInterface $index) { + $location_distance_fields = array(); + + foreach ($index->getFields() as $field) { + if ($field->getType() == 'location') { + $distance_field_name = $field->getFieldIdentifier() . '__distance'; + $distance_field = new Field($index, $distance_field_name); + $distance_field->setLabel($field->getLabel() . ' (distance)'); + $distance_field->setDataDefinition(DataDefinition::create('decimal')); + $distance_field->setType('decimal'); + $distance_field->setDatasourceId($field->getDatasourceId()); + $distance_field->setPropertyPath($distance_field_name); + + $location_distance_fields[$distance_field_name] = $distance_field; + } + } + + return $location_distance_fields; + } + } diff --git a/tests/src/Kernel/SearchApiSolrLocationTest.php b/tests/src/Kernel/SearchApiSolrLocationTest.php new file mode 100644 index 0000000..9b512d4 --- /dev/null +++ b/tests/src/Kernel/SearchApiSolrLocationTest.php @@ -0,0 +1,340 @@ +installConfig([ + 'search_api_solr', + 'search_api_solr_test', + ]); + $this->installEntitySchema('field_storage_config'); + $this->installEntitySchema('field_config'); + + // Create a location field and storage for testing. + FieldStorageConfig::create(array( + 'field_name' => 'location', + 'entity_type' => 'entity_test_mulrev_changed', + 'type' => 'geofield', + ))->save(); + FieldConfig::create([ + 'entity_type' => 'entity_test_mulrev_changed', + 'field_name' => 'location', + 'bundle' => 'item', + ])->save(); + + $this->insertExampleContent(); + + /** @var \Drupal\search_api\Entity\Index $index */ + $index = Index::load($this->indexId); + + $info = array( + 'datasource_id' => 'entity:entity_test_mulrev_changed', + 'property_path' => 'location', + 'type' => 'location', + ); + + $fieldsHelper = $this->container->get('search_api.fields_helper'); + + $index->addField($fieldsHelper->createField($index, 'location', $info)); + $index->save(); + + /** @var \Drupal\search_api\Entity\Server $server */ + $server = Server::load($this->serverId); + + $config = $server->getBackendConfig(); + $config['retrieve_data'] = TRUE; + $server->setBackendConfig($config); + $server->save(); + + $this->detectSolrAvailability(); + + $this->indexItems($this->indexId); + } + + /** + * {@inheritdoc} + */ + protected function indexItems($index_id) { + $index_status = parent::indexItems($index_id); + sleep($this->waitForCommit); + return $index_status; + } + + /** + * Detects the availability of a Solr Server and sets $this->solrAvailable. + */ + protected function detectSolrAvailability() { + // Because this is a kernel test, the routing isn't built by default, so + // we have to force it. + \Drupal::service('router.builder')->rebuild(); + + try { + $backend = Server::load($this->serverId)->getBackend(); + if ($backend->isAvailable()) { + $this->solrAvailable = TRUE; + } + } + catch (\Exception $e) { + } + } + + /** + * {@inheritdoc} + */ + public function insertExampleContent() { + $this->addTestEntity(1, [ + 'name' => 'London', + 'body' => 'London', + 'type' => 'item', + 'location' => 'POINT(-0.076132 51.508530)', + ]); + $this->addTestEntity(2, [ + 'name' => 'New York', + 'body' => 'New York', + 'type' => 'item', + 'location' => 'POINT(-73.138260 40.792240)', + ]); + $this->addTestEntity(3, [ + 'name' => 'Brussels', + 'body' => 'Brussels', + 'type' => 'item', + 'location' => 'POINT(4.355607 50.878899)', + ]); + $count = \Drupal::entityQuery('entity_test_mulrev_changed')->count()->execute(); + $this->assertEquals(3, $count, "$count items inserted."); + } + + /** + * Tests highlight and excerpt options. + */ + public function testBackend() { + + // Search 500km from Antwerp. + $location_options = array( + array( + 'field' => 'location', + 'lat' => '51.260197', + 'lon' => '4.402771', + 'radius' => '500', + ), + ); + /** @var \Drupal\search_api\Query\ResultSet $result */ + $query = $this->buildSearch(NULL, [], NULL, FALSE) + ->sort('location__distance'); + + $query->setOption('search_api_location', $location_options); + $result = $query->execute(); + + $this->assertResults([3, 1], $result, 'Search for 500km from Antwerp ordered by distance'); + + /** @var \Drupal\search_api\Item\Item $item */ + $item = $result->getResultItems()['entity:entity_test_mulrev_changed/3:en']; + $distance = $item->getField('location__distance')->getValues()[0]; + + $this->assertEquals(42.5263374675, $distance, 'The distance is correctly returned'); + + // Search between 100km and 6000km from Antwerp. + $location_options = array( + array( + 'field' => 'location', + 'lat' => '51.260197', + 'lon' => '4.402771', + ), + ); + $query = $this->buildSearch(NULL, [], NULL, FALSE) + ->addCondition('location', array('100', '6000'), 'BETWEEN') + ->sort('location__distance', 'DESC'); + + $query->setOption('search_api_location', $location_options); + $result = $query->execute(); + + $this->assertResults([2, 1], $result, 'Search between 100 and 6000km from Antwerp ordered by distance descending'); + + $facets = array(); + $facets['location__distance'] = array( + 'field' => 'location__distance', + 'limit' => 10, + 'min_count' => 0, + 'missing' => TRUE, + ); + + // Search 1000km from Antwerp. + $location_options = array( + array( + 'field' => 'location', + 'lat' => '51.260197', + 'lon' => '4.402771', + 'radius' => '1000', + ), + ); + $query = $this->buildSearch(NULL, [], NULL, FALSE) + ->sort('location__distance'); + + $query->setOption('search_api_location', $location_options); + $query->setOption('search_api_facets', $facets); + $result = $query->execute(); + + $facets = $result->getExtraData('search_api_facets', array())['location__distance']; + + $expected = [ + [ + 'filter' => '[0 199]', + 'count' => 1, + ], + [ + 'filter' => '[200 399]', + 'count' => 1, + ], + [ + 'filter' => '[400 599]', + 'count' => 0, + ], + [ + 'filter' => '[600 799]', + 'count' => 0, + ], + [ + 'filter' => '[800 999]', + 'count' => 0, + ], + ]; + + $this->assertEquals($expected, $facets, 'The correct location facets are returned'); + + $facets = array(); + $facets['location__distance'] = array( + 'field' => 'location__distance', + 'limit' => 3, + 'min_count' => 1, + 'missing' => TRUE, + ); + + // Search between 100km and 1000km from Antwerp. + $location_options = array( + array( + 'field' => 'location', + 'lat' => '51.260197', + 'lon' => '4.402771', + 'radius' => '1000', + ), + ); + + $query = $this->buildSearch(NULL, [], NULL, FALSE) + ->addCondition('location', array('100', '1000'), 'BETWEEN') + ->sort('location__distance'); + + $query->setOption('search_api_location', $location_options); + $query->setOption('search_api_facets', $facets); + $result = $query->execute(); + + $facets = $result->getExtraData('search_api_facets', array())['location__distance']; + + $expected = [ + [ + 'filter' => '[100 399]', + 'count' => 1, + ], + ]; + + $this->assertEquals($expected, $facets, 'The correct location facets are returned'); + } + + /** + * Tests the correct setup of the server backend. + */ + protected function checkServerBackend() { + // TODO: Implement checkServerBackend() method. + } + + /** + * Checks whether changes to the index's fields are picked up by the server. + */ + protected function updateIndex() { + // TODO: Implement updateIndex() method. + } + + /** + * Tests that a second server doesn't interfere with the first. + */ + protected function checkSecondServer() { + // TODO: Implement checkSecondServer() method. + } + + /** + * Tests whether removing the configuration again works as it should. + */ + protected function checkModuleUninstall() { + // TODO: Implement checkModuleUninstall() method. + } + +}