From 40752c2235913bab1b4e4bcdc8d7befa196eb978 Mon Sep 17 00:00:00 2001 From: alesharik Date: Tue, 31 Mar 2020 21:24:44 +0300 Subject: [PATCH] DATAES-773 - Add search-as-you-type field support. Original PR: #415 --- .../data/elasticsearch/annotations/Field.java | 6 + .../elasticsearch/annotations/FieldType.java | 4 +- .../elasticsearch/annotations/InnerField.java | 6 + .../core/index/MappingParameters.java | 19 ++ .../core/SearchAsYouTypeTests.java | 191 ++++++++++++++++++ 5 files changed, 225 insertions(+), 1 deletion(-) create mode 100644 src/test/java/org/springframework/data/elasticsearch/core/SearchAsYouTypeTests.java diff --git a/src/main/java/org/springframework/data/elasticsearch/annotations/Field.java b/src/main/java/org/springframework/data/elasticsearch/annotations/Field.java index 7adaa7244..2c328c1be 100644 --- a/src/main/java/org/springframework/data/elasticsearch/annotations/Field.java +++ b/src/main/java/org/springframework/data/elasticsearch/annotations/Field.java @@ -33,6 +33,7 @@ import org.springframework.core.annotation.AliasFor; * @author Kevin Leturc * @author Peter-Josef Meisch * @author Xiao Yu + * @author Aleksei Arsenev */ @Retention(RetentionPolicy.RUNTIME) @Target(ElementType.FIELD) @@ -148,4 +149,9 @@ public @interface Field { * @since 4.0 */ double scalingFactor() default 1; + + /** + * @since 4.0 + */ + int maxShingleSize() default -1; } diff --git a/src/main/java/org/springframework/data/elasticsearch/annotations/FieldType.java b/src/main/java/org/springframework/data/elasticsearch/annotations/FieldType.java index 09cc014e4..a2d7088b8 100644 --- a/src/main/java/org/springframework/data/elasticsearch/annotations/FieldType.java +++ b/src/main/java/org/springframework/data/elasticsearch/annotations/FieldType.java @@ -21,6 +21,7 @@ package org.springframework.data.elasticsearch.annotations; * @author Artur Konczak * @author Zeng Zetang * @author Peter-Josef Meisch + * @author Aleksei Arsenev */ public enum FieldType { Auto, @@ -49,5 +50,6 @@ public enum FieldType { Ip, TokenCount, Percolator, - Flattened + Flattened, + Search_As_You_Type } diff --git a/src/main/java/org/springframework/data/elasticsearch/annotations/InnerField.java b/src/main/java/org/springframework/data/elasticsearch/annotations/InnerField.java index c6fa617d7..05edbea2f 100644 --- a/src/main/java/org/springframework/data/elasticsearch/annotations/InnerField.java +++ b/src/main/java/org/springframework/data/elasticsearch/annotations/InnerField.java @@ -26,6 +26,7 @@ import java.lang.annotation.Target; * @author Sascha Woo * @author Xiao Yu * @author Peter-Josef Meisch + * @author Aleksei Arsenev */ @Retention(RetentionPolicy.RUNTIME) @Target(ElementType.FIELD) @@ -117,4 +118,9 @@ public @interface InnerField { * @since 4.0 */ double scalingFactor() default 1; + + /** + * @since 4.0 + */ + int maxShingleSize() default -1; } diff --git a/src/main/java/org/springframework/data/elasticsearch/core/index/MappingParameters.java b/src/main/java/org/springframework/data/elasticsearch/core/index/MappingParameters.java index 39bc8d760..9798e2dfd 100644 --- a/src/main/java/org/springframework/data/elasticsearch/core/index/MappingParameters.java +++ b/src/main/java/org/springframework/data/elasticsearch/core/index/MappingParameters.java @@ -18,6 +18,7 @@ package org.springframework.data.elasticsearch.core.index; import java.io.IOException; import java.lang.annotation.Annotation; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.xcontent.XContentBuilder; import org.springframework.data.elasticsearch.annotations.DateFormat; import org.springframework.data.elasticsearch.annotations.Field; @@ -36,6 +37,7 @@ import org.springframework.util.StringUtils; * {@link org.springframework.data.elasticsearch.annotations.InnerField} annotation. * * @author Peter-Josef Meisch + * @author Aleksei Arsenev * @since 4.0 */ public final class MappingParameters { @@ -64,6 +66,7 @@ public final class MappingParameters { static final String FIELD_PARAM_SIMILARITY = "similarity"; static final String FIELD_PARAM_TERM_VECTOR = "term_vector"; static final String FIELD_PARAM_TYPE = "type"; + static final String FIELD_PARAM_MAX_SHINGLE_SIZE = "max_shingle_size"; private boolean index = true; private boolean store = false; @@ -88,6 +91,8 @@ public final class MappingParameters { private Similarity similarity = Similarity.Default; private TermVector termVector = TermVector.none; private double scalingFactor = 1.0; + @Nullable + private Integer maxShingleSize; /** * extracts the mapping parameters from the relevant annotations. @@ -136,6 +141,11 @@ public final class MappingParameters { similarity = field.similarity(); termVector = field.termVector(); scalingFactor = field.scalingFactor(); + maxShingleSize = field.maxShingleSize() >= 0 ? field.maxShingleSize() : null; + Assert.isTrue(type != FieldType.Search_As_You_Type + || maxShingleSize == null + || (maxShingleSize >= 2 && maxShingleSize <= 4), + "maxShingleSize must be in inclusive range from 2 to 4 for field type search_as_you_type"); } private MappingParameters(InnerField field) { @@ -165,6 +175,11 @@ public final class MappingParameters { similarity = field.similarity(); termVector = field.termVector(); scalingFactor = field.scalingFactor(); + maxShingleSize = field.maxShingleSize() >= 0 ? field.maxShingleSize() : null; + Assert.isTrue(type != FieldType.Search_As_You_Type + || maxShingleSize == null + || (maxShingleSize >= 2 && maxShingleSize <= 4), + "maxShingleSize must be in inclusive range from 2 to 4 for field type search_as_you_type"); } public boolean isStore() { @@ -269,5 +284,9 @@ public final class MappingParameters { if (type == FieldType.Scaled_Float) { builder.field(FIELD_PARAM_SCALING_FACTOR, scalingFactor); } + + if (maxShingleSize != null) { + builder.field(FIELD_PARAM_MAX_SHINGLE_SIZE, maxShingleSize); + } } } diff --git a/src/test/java/org/springframework/data/elasticsearch/core/SearchAsYouTypeTests.java b/src/test/java/org/springframework/data/elasticsearch/core/SearchAsYouTypeTests.java new file mode 100644 index 000000000..5157d441b --- /dev/null +++ b/src/test/java/org/springframework/data/elasticsearch/core/SearchAsYouTypeTests.java @@ -0,0 +1,191 @@ +/* + * Copyright 2014-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.data.elasticsearch.core; + +import lombok.*; +import org.elasticsearch.index.query.MultiMatchQueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.data.annotation.Id; +import org.springframework.data.elasticsearch.annotations.Document; +import org.springframework.data.elasticsearch.annotations.Field; +import org.springframework.data.elasticsearch.annotations.FieldType; +import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates; +import org.springframework.data.elasticsearch.core.query.IndexQuery; +import org.springframework.data.elasticsearch.core.query.NativeSearchQuery; +import org.springframework.data.elasticsearch.core.query.Query; +import org.springframework.data.elasticsearch.junit.jupiter.ElasticsearchRestTemplateConfiguration; +import org.springframework.data.elasticsearch.junit.jupiter.SpringIntegrationTest; +import org.springframework.data.elasticsearch.utils.IndexInitializer; +import org.springframework.lang.Nullable; +import org.springframework.test.context.ContextConfiguration; + +import javax.annotation.Nonnull; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * @author Aleksei Arsenev + */ +@SpringIntegrationTest +@ContextConfiguration(classes = { SearchAsYouTypeTests.Config.class }) +public class SearchAsYouTypeTests { + @Configuration + @Import({ ElasticsearchRestTemplateConfiguration.class }) + static class Config {} + @Autowired + private ElasticsearchOperations operations; + + @BeforeEach + private void setup() { + IndexInitializer.init(operations.indexOps(SearchAsYouTypeEntity.class)); + } + + @AfterEach + void after() { + operations.indexOps(SearchAsYouTypeEntity.class).delete(); + } + + private void loadEntities() { + List indexQueries = new ArrayList<>(); + indexQueries.add(SearchAsYouTypeEntity.builder().id("1").name("test 1").suggest("test 1234").build().toIndex()); + indexQueries.add(SearchAsYouTypeEntity.builder().id("2").name("test 2").suggest("test 5678").build().toIndex()); + indexQueries.add(SearchAsYouTypeEntity.builder().id("3").name("test 3").suggest("asd 5678").build().toIndex()); + indexQueries.add(SearchAsYouTypeEntity.builder().id("4").name("test 4").suggest("not match").build().toIndex()); + IndexCoordinates index = IndexCoordinates.of("test-index-core-search-as-you-type"); + operations.bulkIndex(indexQueries, index); + operations.indexOps(SearchAsYouTypeEntity.class).refresh(); + } + + @Test // DATAES-773 + void shouldRetrieveEntityById() { + loadEntities(); + IndexCoordinates index = IndexCoordinates.of("test-index-core-search-as-you-type"); + operations.get("1", SearchAsYouTypeEntity.class, index); + } + + @Test // DATAES-773 + void shouldReturnCorrectResultsForTextString() { + + // given + loadEntities(); + + // when + Query query = new NativeSearchQuery(QueryBuilders.multiMatchQuery("test ", // + "suggest", "suggest._2gram", "suggest._3gram", "suggest._4gram") + .type(MultiMatchQueryBuilder.Type.BOOL_PREFIX)); + IndexCoordinates index = IndexCoordinates.of("test-index-core-search-as-you-type"); + List result = operations.search(query, SearchAsYouTypeEntity.class, index) // + .getSearchHits() // + .stream() // + .map(SearchHit::getContent) // + .collect(Collectors.toList()); + + // then + assertEquals(2, result.size()); + assertTrue(result.contains(new SearchAsYouTypeEntity("1"))); + assertTrue(result.contains(new SearchAsYouTypeEntity("2"))); + } + + @Test // DATAES-773 + void shouldReturnCorrectResultsForNumQuery() { + + // given + loadEntities(); + + // when + Query query = new NativeSearchQuery(QueryBuilders.multiMatchQuery("5678 ", // + "suggest", "suggest._2gram", "suggest._3gram", "suggest._4gram") + .type(MultiMatchQueryBuilder.Type.BOOL_PREFIX)); + IndexCoordinates index = IndexCoordinates.of("test-index-core-search-as-you-type"); + List result = operations.search(query, SearchAsYouTypeEntity.class, index) // + .getSearchHits() // + .stream() // + .map(SearchHit::getContent) // + .collect(Collectors.toList()); + + // then + assertEquals(2, result.size()); + assertTrue(result.contains(new SearchAsYouTypeEntity("2"))); + assertTrue(result.contains(new SearchAsYouTypeEntity("3"))); + } + + @Test // DATAES-773 + void shouldReturnCorrectResultsForNotMatchQuery() { + + // given + loadEntities(); + + // when + Query query = new NativeSearchQuery(QueryBuilders.multiMatchQuery("n mat", // + "suggest", "suggest._2gram", "suggest._3gram", "suggest._4gram") + .type(MultiMatchQueryBuilder.Type.BOOL_PREFIX)); + IndexCoordinates index = IndexCoordinates.of("test-index-core-search-as-you-type"); + List result = operations.search(query, SearchAsYouTypeEntity.class, index) // + .getSearchHits() // + .stream() // + .map(SearchHit::getContent) // + .collect(Collectors.toList()); + + // then + assertEquals(1, result.size()); + assertTrue(result.contains(new SearchAsYouTypeEntity("4"))); + } + + /** + * @author Aleksei Arsenev + */ + @Data + @Builder + @NoArgsConstructor + @AllArgsConstructor + @EqualsAndHashCode(onlyExplicitlyIncluded = true) + @Document(indexName = "test-index-core-search-as-you-type", replicas = 0, refreshInterval = "-1") + static class SearchAsYouTypeEntity { + + public SearchAsYouTypeEntity(@Nonnull String id) { + this.id = id; + } + + @Nonnull + @Id + @EqualsAndHashCode.Include + private String id; + + @Nullable + private String name; + + @Nullable + @Field(type = FieldType.Search_As_You_Type, maxShingleSize = 4) + private String suggest; + + public IndexQuery toIndex() { + IndexQuery indexQuery = new IndexQuery(); + indexQuery.setId(getId()); + indexQuery.setObject(this); + return indexQuery; + } + } +}