DATAES-973 - Release version 4.1.2 (2020.0.2).

Original PR: #557

(cherry picked from commit 54727229e1)

Jenkinsfile

@@ -3,7 +3,7 @@ pipeline {
 
 	triggers {
 		pollSCM 'H/10 * * * *'
-		upstream(upstreamProjects: "spring-data-commons/master", threshold: hudson.model.Result.SUCCESS)
+		upstream(upstreamProjects: "spring-data-commons/2.4.x", threshold: hudson.model.Result.SUCCESS)
 	}
 
 	options {
@@ -15,7 +15,7 @@ pipeline {
 		stage("test: baseline (jdk8)") {
 			when {
 				anyOf {
-					branch 'master'
+					branch '4.1.x'
 					not { triggeredBy 'UpstreamCause' }
 				}
 			}
@@ -23,24 +23,21 @@ pipeline {
 				docker {
 					image 'adoptopenjdk/openjdk8:latest'
 					label 'data'
-					args '-v $HOME:/tmp/jenkins-home'
-					args '-u root -v /var/run/docker.sock:/var/run/docker.sock'
+					args '-u root -v /var/run/docker.sock:/var/run/docker.sock -v $HOME:/tmp/jenkins-home'
 				}
 			}
 			options { timeout(time: 30, unit: 'MINUTES') }
 			steps {
 			    sh 'mkdir -p /tmp/jenkins-home'
             	sh 'chown -R 1001:1001 .'
-				sh 'rm -rf ?'
-				sh 'MAVEN_OPTS="-Duser.name=jenkins -Duser.home=/tmp/jenkins-home" ./mvnw clean dependency:list verify -Dsort -U -B'
-            	sh 'chown -R 1001:1001 .'
+				sh 'MAVEN_OPTS="-Duser.name=jenkins -Duser.home=/tmp/jenkins-home" ./mvnw clean dependency:list verify -Dsort -U -B -Dmaven.repo.local=/tmp/jenkins-home/.m2/spring-data-elasticsearch'
 			}
 		}
 
 		stage("Test other configurations") {
 			when {
 				allOf {
-					branch 'master'
+					branch '4.1.x'
 					not { triggeredBy 'UpstreamCause' }
 				}
 			}
@@ -50,17 +47,12 @@ pipeline {
 						docker {
 							image 'adoptopenjdk/openjdk11:latest'
 							label 'data'
-							args '-v $HOME:/tmp/jenkins-home'
-        					args '-u root -v /var/run/docker.sock:/var/run/docker.sock'
+							args '-u root -v /var/run/docker.sock:/var/run/docker.sock -v $HOME:/tmp/jenkins-home'
 						}
 					}
 					options { timeout(time: 30, unit: 'MINUTES') }
 					steps {
-                        sh 'mkdir -p /tmp/jenkins-home'
-                        sh 'chown -R 1001:1001 .'
-						sh 'rm -rf ?'
-						sh 'MAVEN_OPTS="-Duser.name=jenkins -Duser.home=/tmp/jenkins-home" ./mvnw -Pjava11 clean dependency:list verify -Dsort -U -B'
-                        sh 'chown -R 1001:1001 .'
+						sh 'MAVEN_OPTS="-Duser.name=jenkins -Duser.home=/tmp/jenkins-home" ./mvnw -Pjava11 clean dependency:list verify -Dsort -U -B -Dmaven.repo.local=/tmp/jenkins-home/.m2/spring-data-elasticsearch'
 					}
 				}
 
@@ -69,17 +61,12 @@ pipeline {
 						docker {
 							image 'adoptopenjdk/openjdk15:latest'
 							label 'data'
-							args '-v $HOME:/tmp/jenkins-home'
-        					args '-u root -v /var/run/docker.sock:/var/run/docker.sock'
+        					args '-u root -v /var/run/docker.sock:/var/run/docker.sock -v $HOME:/tmp/jenkins-home'
 						}
 					}
 					options { timeout(time: 30, unit: 'MINUTES') }
 					steps {
-                        sh 'mkdir -p /tmp/jenkins-home'
-                        sh 'chown -R 1001:1001 .'
-						sh 'rm -rf ?'
-						sh 'MAVEN_OPTS="-Duser.name=jenkins -Duser.home=/tmp/jenkins-home" ./mvnw -Pjava11 clean dependency:list verify -Dsort -U -B'
-                        sh 'chown -R 1001:1001 .'
+						sh 'MAVEN_OPTS="-Duser.name=jenkins -Duser.home=/tmp/jenkins-home" ./mvnw -Pjava11 clean dependency:list verify -Dsort -U -B -Dmaven.repo.local=/tmp/jenkins-home/.m2/spring-data-elasticsearch'
 					}
 				}
 			}
@@ -88,7 +75,7 @@ pipeline {
 		stage('Release to artifactory') {
 			when {
 				anyOf {
-					branch 'master'
+					branch '4.1.x'
 					not { triggeredBy 'UpstreamCause' }
 				}
 			}
@@ -96,7 +83,7 @@ pipeline {
 				docker {
 					image 'adoptopenjdk/openjdk8:latest'
 					label 'data'
-					args '-v $HOME:/tmp/jenkins-home'
+					args '-u root -v /var/run/docker.sock:/var/run/docker.sock -v $HOME:/tmp/jenkins-home'
 				}
 			}
 			options { timeout(time: 20, unit: 'MINUTES') }
@@ -106,8 +93,7 @@ pipeline {
 			}
 
 			steps {
-				sh 'rm -rf ?'
-				sh 'MAVEN_OPTS="-Duser.name=jenkins -Duser.home=/tmp/jenkins-home" ./mvnw -Pci,artifactory ' +
+				sh 'MAVEN_OPTS="-Duser.name=jenkins -Duser.home=/tmp/jenkins-home" ./mvnw -Pci,artifactory -Dmaven.repo.local=/tmp/jenkins-home/.m2/spring-data-elasticsearch ' +
 						'-Dartifactory.server=https://repo.spring.io ' +
 						"-Dartifactory.username=${ARTIFACTORY_USR} " +
 						"-Dartifactory.password=${ARTIFACTORY_PSW} " +
@@ -119,13 +105,13 @@ pipeline {
 		}
 		stage('Publish documentation') {
 			when {
-				branch 'master'
+				branch '4.1.x'
 			}
 			agent {
 				docker {
 					image 'adoptopenjdk/openjdk8:latest'
 					label 'data'
-					args '-v $HOME:/tmp/jenkins-home'
+					args '-u root -v /var/run/docker.sock:/var/run/docker.sock -v $HOME:/tmp/jenkins-home'
 				}
 			}
 			options { timeout(time: 20, unit: 'MINUTES') }
@@ -135,7 +121,7 @@ pipeline {
 			}
 
 			steps {
-				sh 'MAVEN_OPTS="-Duser.name=jenkins -Duser.home=/tmp/jenkins-home" ./mvnw -Pci,distribute ' +
+				sh 'MAVEN_OPTS="-Duser.name=jenkins -Duser.home=/tmp/jenkins-home" ./mvnw -Pci,distribute -Dmaven.repo.local=/tmp/jenkins-home/.m2/spring-data-elasticsearch ' +
 						'-Dartifactory.server=https://repo.spring.io ' +
 						"-Dartifactory.username=${ARTIFACTORY_USR} " +
 						"-Dartifactory.password=${ARTIFACTORY_PSW} " +

pom.xml

@@ -5,12 +5,12 @@
 
 	<groupId>org.springframework.data</groupId>
 	<artifactId>spring-data-elasticsearch</artifactId>
-	<version>4.1.0</version>
+	<version>4.1.2</version>
 
 	<parent>
 		<groupId>org.springframework.data.build</groupId>
 		<artifactId>spring-data-parent</artifactId>
-		<version>2.4.0</version>
+		<version>2.4.2</version>
 	</parent>
 
 	<name>Spring Data Elasticsearch</name>
@@ -22,7 +22,7 @@
 		<elasticsearch>7.9.3</elasticsearch>
 		<log4j>2.13.3</log4j>
 		<netty>4.1.52.Final</netty>
-		<springdata.commons>2.4.0</springdata.commons>
+		<springdata.commons>2.4.2</springdata.commons>
 		<testcontainers>1.14.3</testcontainers>
 		<java-module-name>spring.data.elasticsearch</java-module-name>
 	</properties>

src/main/asciidoc/preface.adoc

@@ -34,7 +34,7 @@ The following table shows the Elasticsearch versions that are used by Spring Dat
 [cols="^,^,^,^",options="header"]
 |===
 | Spring Data Release Train |Spring Data Elasticsearch |Elasticsearch | Spring Boot
-| 2020.0.0footnote:cdv[Currently in development] |4.1.xfootnote:cdv[]|7.9.3 |2.3.xfootnote:cdv[]
+| 2020.0.0footnote:cdv[Currently in development] |4.1.xfootnote:cdv[]|7.9.3 |2.4.xfootnote:cdv[]
 | Neumann | 4.0.x | 7.6.2 |2.3.x
 | Moore | 3.2.x |6.8.12 | 2.2.x
 | Lovelace | 3.1.x | 6.2.2 |2.1.x

src/main/asciidoc/reference/elasticsearch-object-mapping.adoc

@@ -58,13 +58,14 @@ Constructor arguments are mapped by name to the key values in the retrieved Docu
 ** `name`: The name of the field as it will be represented in the Elasticsearch document, if not set, the Java field name is used.
 ** `type`: the field type, can be one of _Text, Keyword, Long, Integer, Short, Byte, Double, Float, Half_Float, Scaled_Float, Date, Date_Nanos, Boolean, Binary, Integer_Range, Float_Range, Long_Range, Double_Range, Date_Range, Ip_Range, Object, Nested, Ip, TokenCount, Percolator, Flattened, Search_As_You_Type_.
 See https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-types.html[Elasticsearch Mapping Types]
-** `format` and `pattern` definitions for the _Date_ type. `format` must be defined for date types.
+** `format` and `pattern` definitions for the _Date_ type.
 ** `store`: Flag whether the original field value should be store in Elasticsearch, default value is _false_.
 ** `analyzer`, `searchAnalyzer`, `normalizer` for specifying custom analyzers and normalizer.
 * `@GeoPoint`: marks a field as _geo_point_ datatype.
 Can be omitted if the field is an instance of the `GeoPoint` class.
 
-NOTE: Properties that derive from `TemporalAccessor` must either have a `@Field` annotation of type `FieldType.Date` or a custom converter must be registered for this type. +
+NOTE: Properties that derive from `TemporalAccessor` or are of type `java.util.Date` must either have a `@Field` annotation of type `FieldType.Date` and a
+format different from `DateFormat.none` or a custom converter must be registered for this type. +
 If you are using a custom date format, you need to use _uuuu_ for the year instead of _yyyy_.
 This is due to a https://www.elastic.co/guide/en/elasticsearch/reference/current/migrate-to-java-time.html#java-time-migration-incompatible-date-formats[change in Elasticsearch 7].
 

src/main/java/org/springframework/data/elasticsearch/annotations/TermVector.java

@@ -20,5 +20,5 @@ package org.springframework.data.elasticsearch.annotations;
  * @since 4.0
  */
 public enum TermVector {
-	none, no, yes, with_positions, with_offsets, with_positions_offsets, with_positions_payloads, with_positions_offets_payloads
+	none, no, yes, with_positions, with_offsets, with_positions_offsets, with_positions_payloads, with_positions_offsets_payloads
 }

src/main/java/org/springframework/data/elasticsearch/config/AbstractElasticsearchConfiguration.java

@@ -31,20 +31,25 @@ public abstract class AbstractElasticsearchConfiguration extends ElasticsearchCo
 
 	/**
 	 * Return the {@link RestHighLevelClient} instance used to connect to the cluster. <br />
-	 * Annotate with {@link Bean} in case you want to expose a {@link RestHighLevelClient} instance to the
-	 * {@link org.springframework.context.ApplicationContext}.
 	 *
 	 * @return never {@literal null}.
 	 */
+	@Bean
 	public abstract RestHighLevelClient elasticsearchClient();
 
-	/**
-	 * Creates {@link ElasticsearchOperations}.
-	 *
-	 * @return never {@literal null}.
-	 */
+    /**
+     * Creates {@link ElasticsearchOperations}. <br/>
+     * NOTE: in version 4.1.2 the second parameter was added, previously this implementation called
+     * {@link #elasticsearchClient()} directly. This is not possible anymore, as the base configuration classes don not
+     * use proxied bean methods anymore.
+     *
+     * @param elasticsearchConverter the {@link ElasticsearchConverter} to use*
+     * @param elasticsearchClient the {@link RestHighLevelClient} to use
+     * @return never {@literal null}.
+     */
 	@Bean(name = { "elasticsearchOperations", "elasticsearchTemplate" })
-	public ElasticsearchOperations elasticsearchOperations(ElasticsearchConverter elasticsearchConverter) {
-		return new ElasticsearchRestTemplate(elasticsearchClient(), elasticsearchConverter);
+	public ElasticsearchOperations elasticsearchOperations(ElasticsearchConverter elasticsearchConverter,
+			RestHighLevelClient elasticsearchClient) {
+		return new ElasticsearchRestTemplate(elasticsearchClient, elasticsearchConverter);
 	}
 }

src/main/java/org/springframework/data/elasticsearch/config/AbstractReactiveElasticsearchConfiguration.java

@@ -18,7 +18,6 @@ package org.springframework.data.elasticsearch.config;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.WriteRequest.RefreshPolicy;
 import org.springframework.context.annotation.Bean;
-import org.springframework.context.annotation.Configuration;
 import org.springframework.data.elasticsearch.client.reactive.ReactiveElasticsearchClient;
 import org.springframework.data.elasticsearch.core.ReactiveElasticsearchOperations;
 import org.springframework.data.elasticsearch.core.ReactiveElasticsearchTemplate;
@@ -31,16 +30,14 @@ import org.springframework.lang.Nullable;
  * @since 3.2
  * @see ElasticsearchConfigurationSupport
  */
-@Configuration
 public abstract class AbstractReactiveElasticsearchConfiguration extends ElasticsearchConfigurationSupport {
 
 	/**
 	 * Return the {@link ReactiveElasticsearchClient} instance used to connect to the cluster. <br />
-	 * Annotate with {@link Bean} in case you want to expose a {@link ReactiveElasticsearchClient} instance to the
-	 * {@link org.springframework.context.ApplicationContext}.
 	 *
 	 * @return never {@literal null}.
 	 */
+	@Bean
 	public abstract ReactiveElasticsearchClient reactiveElasticsearchClient();
 
 	/**
@@ -49,9 +46,10 @@ public abstract class AbstractReactiveElasticsearchConfiguration extends Elastic
 	 * @return never {@literal null}.
 	 */
 	@Bean
-	public ReactiveElasticsearchOperations reactiveElasticsearchTemplate(ElasticsearchConverter elasticsearchConverter) {
+	public ReactiveElasticsearchOperations reactiveElasticsearchTemplate(ElasticsearchConverter elasticsearchConverter,
+			ReactiveElasticsearchClient reactiveElasticsearchClient) {
 
-		ReactiveElasticsearchTemplate template = new ReactiveElasticsearchTemplate(reactiveElasticsearchClient(),
+		ReactiveElasticsearchTemplate template = new ReactiveElasticsearchTemplate(reactiveElasticsearchClient,
 				elasticsearchConverter);
 		template.setIndicesOptions(indicesOptions());
 		template.setRefreshPolicy(refreshPolicy());

src/main/java/org/springframework/data/elasticsearch/config/ElasticsearchConfigurationSupport.java

@@ -40,15 +40,16 @@ import org.springframework.util.StringUtils;
  * @author Peter-Josef Meisch
  * @since 3.2
  */
-@Configuration
+@Configuration(proxyBeanMethods = false)
 public class ElasticsearchConfigurationSupport {
 
 	@Bean
 	public ElasticsearchConverter elasticsearchEntityMapper(
-			SimpleElasticsearchMappingContext elasticsearchMappingContext) {
+			SimpleElasticsearchMappingContext elasticsearchMappingContext, ElasticsearchCustomConversions elasticsearchCustomConversions) {
+
 		MappingElasticsearchConverter elasticsearchConverter = new MappingElasticsearchConverter(
 				elasticsearchMappingContext);
-		elasticsearchConverter.setConversions(elasticsearchCustomConversions());
+		elasticsearchConverter.setConversions(elasticsearchCustomConversions);
 		return elasticsearchConverter;
 	}
 
@@ -60,11 +61,11 @@ public class ElasticsearchConfigurationSupport {
 	 * @return never {@literal null}.
 	 */
 	@Bean
-	public SimpleElasticsearchMappingContext elasticsearchMappingContext() {
+	public SimpleElasticsearchMappingContext elasticsearchMappingContext(ElasticsearchCustomConversions elasticsearchCustomConversions) {
 
 		SimpleElasticsearchMappingContext mappingContext = new SimpleElasticsearchMappingContext();
 		mappingContext.setInitialEntitySet(getInitialEntitySet());
-		mappingContext.setSimpleTypeHolder(elasticsearchCustomConversions().getSimpleTypeHolder());
+		mappingContext.setSimpleTypeHolder(elasticsearchCustomConversions.getSimpleTypeHolder());
 
 		return mappingContext;
 	}

src/main/java/org/springframework/data/elasticsearch/core/AbstractElasticsearchTemplate.java

@@ -147,13 +147,14 @@ public abstract class AbstractElasticsearchTemplate implements ElasticsearchOper
 		Assert.notNull(entity, "entity must not be null");
 		Assert.notNull(index, "index must not be null");
 
-		IndexQuery query = getIndexQuery(entity);
-		index(query, index);
+		T entityAfterBeforeConvert = maybeCallbackBeforeConvert(entity, index);
 
-		// suppressing because it's either entity itself or something of a correct type returned by an entity callback
-		@SuppressWarnings("unchecked")
-		T castResult = (T) query.getObject();
-		return castResult;
+		IndexQuery query = getIndexQuery(entityAfterBeforeConvert);
+		doIndex(query, index);
+
+		T entityAfterAfterSave = maybeCallbackAfterSave(entityAfterBeforeConvert, index);
+
+		return entityAfterAfterSave;
 	}
 
 	@Override
@@ -192,6 +193,20 @@ public abstract class AbstractElasticsearchTemplate implements ElasticsearchOper
 		return save(Arrays.asList(entities));
 	}
 
+	@Override
+	public String index(IndexQuery query, IndexCoordinates index) {
+
+		maybeCallbackBeforeConvertWithQuery(query, index);
+
+		String documentId = doIndex(query, index);
+
+		maybeCallbackAfterSaveWithQuery(query, index);
+
+		return documentId;
+	}
+
+	public abstract String doIndex(IndexQuery query, IndexCoordinates indexCoordinates);
+
 	@Override
 	@Nullable
 	public <T> T get(String id, Class<T> clazz) {
@@ -261,11 +276,38 @@ public abstract class AbstractElasticsearchTemplate implements ElasticsearchOper
 		return bulkIndex(queries, bulkOptions, getIndexCoordinatesFor(clazz));
 	}
 
+	@Override
+	public final List<IndexedObjectInformation> bulkIndex(List<IndexQuery> queries, BulkOptions bulkOptions,
+			IndexCoordinates index) {
+
+		Assert.notNull(queries, "List of IndexQuery must not be null");
+		Assert.notNull(bulkOptions, "BulkOptions must not be null");
+
+		return bulkOperation(queries, bulkOptions, index);
+	}
+
 	@Override
 	public void bulkUpdate(List<UpdateQuery> queries, Class<?> clazz) {
 		bulkUpdate(queries, getIndexCoordinatesFor(clazz));
 	}
 
+	public List<IndexedObjectInformation> bulkOperation(List<?> queries, BulkOptions bulkOptions,
+			IndexCoordinates index) {
+
+		Assert.notNull(queries, "List of IndexQuery must not be null");
+		Assert.notNull(bulkOptions, "BulkOptions must not be null");
+
+		maybeCallbackBeforeConvertWithQueries(queries, index);
+
+		List<IndexedObjectInformation> indexedObjectInformations = doBulkOperation(queries, bulkOptions, index);
+
+		maybeCallbackAfterSaveWithQueries(queries, index);
+
+		return indexedObjectInformations;
+	}
+
+	public abstract List<IndexedObjectInformation> doBulkOperation(List<?> queries, BulkOptions bulkOptions,
+			IndexCoordinates index);
 	// endregion
 
 	// region SearchOperations
@@ -620,6 +662,20 @@ public abstract class AbstractElasticsearchTemplate implements ElasticsearchOper
 			if (queryObject != null) {
 				queryObject = maybeCallbackBeforeConvert(queryObject, index);
 				indexQuery.setObject(queryObject);
+				// the callback might have set som values relevant for the IndexQuery
+				IndexQuery newQuery = getIndexQuery(queryObject);
+
+				if (indexQuery.getRouting() == null && newQuery.getRouting() != null) {
+					indexQuery.setRouting(newQuery.getRouting());
+				}
+
+				if (indexQuery.getSeqNo() == null && newQuery.getSeqNo() != null) {
+					indexQuery.setSeqNo(newQuery.getSeqNo());
+				}
+
+				if (indexQuery.getPrimaryTerm() == null && newQuery.getPrimaryTerm() != null) {
+					indexQuery.setPrimaryTerm(newQuery.getPrimaryTerm());
+				}
 			}
 		}
 	}

src/main/java/org/springframework/data/elasticsearch/core/DefaultIndexOperations.java

@@ -16,6 +16,7 @@
 package org.springframework.data.elasticsearch.core;
 
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -34,11 +35,13 @@ import org.elasticsearch.client.indices.GetIndexRequest;
 import org.elasticsearch.client.indices.GetIndexTemplatesRequest;
 import org.elasticsearch.client.indices.GetIndexTemplatesResponse;
 import org.elasticsearch.client.indices.GetMappingsRequest;
-import org.elasticsearch.client.indices.GetMappingsResponse;
 import org.elasticsearch.client.indices.IndexTemplatesExistRequest;
 import org.elasticsearch.client.indices.PutIndexTemplateRequest;
 import org.elasticsearch.client.indices.PutMappingRequest;
 import org.elasticsearch.cluster.metadata.AliasMetadata;
+import org.elasticsearch.cluster.metadata.MappingMetadata;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.springframework.data.elasticsearch.core.document.Document;
 import org.springframework.data.elasticsearch.core.index.AliasActions;
 import org.springframework.data.elasticsearch.core.index.AliasData;
@@ -61,6 +64,8 @@ import org.springframework.util.Assert;
  */
 class DefaultIndexOperations extends AbstractDefaultIndexOperations implements IndexOperations {
 
+	private static final Logger LOGGER = LoggerFactory.getLogger(DefaultIndexOperations.class);
+
 	private final ElasticsearchRestTemplate restTemplate;
 
 	public DefaultIndexOperations(ElasticsearchRestTemplate restTemplate, Class<?> boundClass) {
@@ -117,10 +122,19 @@ class DefaultIndexOperations extends AbstractDefaultIndexOperations implements I
 		GetMappingsRequest mappingsRequest = requestFactory.getMappingsRequest(index);
 
 		return restTemplate.execute(client -> {
-			GetMappingsResponse mapping = client.indices().getMapping(mappingsRequest, RequestOptions.DEFAULT);
-			// we only return data for the first index name that was requested (always have done so)
-			String index1 = mappingsRequest.indices()[0];
-			return mapping.mappings().get(index1).getSourceAsMap();
+			Map<String, MappingMetadata> mappings = client.indices() //
+					.getMapping(mappingsRequest, RequestOptions.DEFAULT) //
+					.mappings(); //
+
+			if (mappings == null || mappings.size() == 0) {
+				return Collections.emptyMap();
+			}
+
+			if (mappings.size() > 1) {
+				LOGGER.warn("more than one mapping returned for " + index.getIndexName());
+			}
+			// we have at least one, take the first from the iterator
+			return mappings.entrySet().iterator().next().getValue().getSourceAsMap();
 		});
 	}
 

src/main/java/org/springframework/data/elasticsearch/core/DefaultTransportIndexOperations.java

@@ -15,6 +15,7 @@
  */
 package org.springframework.data.elasticsearch.core;
 
+import java.util.Collections;
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.LinkedHashMap;
@@ -40,6 +41,7 @@ import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateReque
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.metadata.AliasMetadata;
 import org.elasticsearch.cluster.metadata.IndexTemplateMetadata;
+import org.elasticsearch.cluster.metadata.MappingMetadata;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.settings.Settings;
@@ -126,10 +128,19 @@ class DefaultTransportIndexOperations extends AbstractDefaultIndexOperations imp
 
 		GetMappingsRequest mappingsRequest = requestFactory.getMappingsRequest(client, index);
 
-		return client.admin().indices().getMappings( //
+		ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetadata>> mappings = client.admin().indices().getMappings( //
 				mappingsRequest).actionGet() //
-				.getMappings().get(mappingsRequest.indices()[0]).get(IndexCoordinates.TYPE) //
-				.getSourceAsMap();
+				.getMappings();
+
+		if (mappings == null || mappings.size() == 0) {
+			return Collections.emptyMap();
+		}
+
+		if (mappings.size() > 1) {
+			LOGGER.warn("more than one mapping returned for " + index.getIndexName());
+		}
+		// we have at least one, take the first from the iterator
+		return mappings.iterator().next().value.get(IndexCoordinates.TYPE).getSourceAsMap();
 	}
 
 	@Override

src/main/java/org/springframework/data/elasticsearch/core/ElasticsearchRestTemplate.java

@@ -137,10 +137,7 @@ public class ElasticsearchRestTemplate extends AbstractElasticsearchTemplate {
 	// endregion
 
 	// region DocumentOperations
-	@Override
-	public String index(IndexQuery query, IndexCoordinates index) {
-
-		maybeCallbackBeforeConvertWithQuery(query, index);
+	public String doIndex(IndexQuery query, IndexCoordinates index) {
 
 		IndexRequest request = requestFactory.indexRequest(query, index);
 		IndexResponse indexResponse = execute(client -> client.index(request, RequestOptions.DEFAULT));
@@ -152,8 +149,6 @@ public class ElasticsearchRestTemplate extends AbstractElasticsearchTemplate {
 					indexResponse.getPrimaryTerm(), indexResponse.getVersion()));
 		}
 
-		maybeCallbackAfterSaveWithQuery(query, index);
-
 		return indexResponse.getId();
 	}
 
@@ -187,16 +182,6 @@ public class ElasticsearchRestTemplate extends AbstractElasticsearchTemplate {
 		return execute(client -> client.get(request, RequestOptions.DEFAULT).isExists());
 	}
 
-	@Override
-	public List<IndexedObjectInformation> bulkIndex(List<IndexQuery> queries, BulkOptions bulkOptions,
-			IndexCoordinates index) {
-
-		Assert.notNull(queries, "List of IndexQuery must not be null");
-		Assert.notNull(bulkOptions, "BulkOptions must not be null");
-
-		return doBulkOperation(queries, bulkOptions, index);
-	}
-
 	@Override
 	public void bulkUpdate(List<UpdateQuery> queries, BulkOptions bulkOptions, IndexCoordinates index) {
 
@@ -237,14 +222,12 @@ public class ElasticsearchRestTemplate extends AbstractElasticsearchTemplate {
 		return new UpdateResponse(result);
 	}
 
-	private List<IndexedObjectInformation> doBulkOperation(List<?> queries, BulkOptions bulkOptions,
+	public List<IndexedObjectInformation> doBulkOperation(List<?> queries, BulkOptions bulkOptions,
 			IndexCoordinates index) {
-		maybeCallbackBeforeConvertWithQueries(queries, index);
 		BulkRequest bulkRequest = requestFactory.bulkRequest(queries, bulkOptions, index);
 		List<IndexedObjectInformation> indexedObjectInformationList = checkForBulkOperationFailure(
 				execute(client -> client.bulk(bulkRequest, RequestOptions.DEFAULT)));
 		updateIndexedObjectsWithQueries(queries, indexedObjectInformationList);
-		maybeCallbackAfterSaveWithQueries(queries, index);
 		return indexedObjectInformationList;
 	}
 	// endregion

src/main/java/org/springframework/data/elasticsearch/core/ElasticsearchTemplate.java

@@ -144,10 +144,7 @@ public class ElasticsearchTemplate extends AbstractElasticsearchTemplate {
 	// endregion
 
 	// region DocumentOperations
-	@Override
-	public String index(IndexQuery query, IndexCoordinates index) {
-
-		maybeCallbackBeforeConvertWithQuery(query, index);
+	public String doIndex(IndexQuery query, IndexCoordinates index) {
 
 		IndexRequestBuilder indexRequestBuilder = requestFactory.indexRequestBuilder(client, query, index);
 		ActionFuture<IndexResponse> future = indexRequestBuilder.execute();
@@ -166,8 +163,6 @@ public class ElasticsearchTemplate extends AbstractElasticsearchTemplate {
 					response.getPrimaryTerm(), response.getVersion()));
 		}
 
-		maybeCallbackAfterSaveWithQuery(query, index);
-
 		return documentId;
 	}
 
@@ -201,22 +196,6 @@ public class ElasticsearchTemplate extends AbstractElasticsearchTemplate {
 		return getRequestBuilder.execute().actionGet().isExists();
 	}
 
-	@Override
-	public List<IndexedObjectInformation> bulkIndex(List<IndexQuery> queries, BulkOptions bulkOptions,
-			IndexCoordinates index) {
-
-		Assert.notNull(queries, "List of IndexQuery must not be null");
-		Assert.notNull(bulkOptions, "BulkOptions must not be null");
-
-		List<IndexedObjectInformation> indexedObjectInformations = doBulkOperation(queries, bulkOptions, index);
-
-		updateIndexedObjectsWithQueries(queries, indexedObjectInformations);
-
-		maybeCallbackAfterSaveWithQueries(queries, index);
-
-		return indexedObjectInformations;
-	}
-
 	@Override
 	public void bulkUpdate(List<UpdateQuery> queries, BulkOptions bulkOptions, IndexCoordinates index) {
 
@@ -261,11 +240,13 @@ public class ElasticsearchTemplate extends AbstractElasticsearchTemplate {
 		return new UpdateResponse(result);
 	}
 
-	private List<IndexedObjectInformation> doBulkOperation(List<?> queries, BulkOptions bulkOptions,
+	public List<IndexedObjectInformation> doBulkOperation(List<?> queries, BulkOptions bulkOptions,
 			IndexCoordinates index) {
-		maybeCallbackBeforeConvertWithQueries(queries, index);
 		BulkRequestBuilder bulkRequest = requestFactory.bulkRequestBuilder(client, queries, bulkOptions, index);
-		return checkForBulkOperationFailure(bulkRequest.execute().actionGet());
+		final List<IndexedObjectInformation> indexedObjectInformations = checkForBulkOperationFailure(
+				bulkRequest.execute().actionGet());
+		updateIndexedObjectsWithQueries(queries, indexedObjectInformations);
+		return indexedObjectInformations;
 	}
 	// endregion
 

src/main/java/org/springframework/data/elasticsearch/core/mapping/SimpleElasticsearchPersistentEntity.java

@@ -31,6 +31,9 @@ import org.springframework.data.mapping.MappingException;
 import org.springframework.data.mapping.PropertyHandler;
 import org.springframework.data.mapping.model.BasicPersistentEntity;
 import org.springframework.data.mapping.model.PersistentPropertyAccessorFactory;
+import org.springframework.data.spel.EvaluationContextProvider;
+import org.springframework.data.spel.ExpressionDependencies;
+import org.springframework.data.util.Lazy;
 import org.springframework.data.util.TypeInformation;
 import org.springframework.expression.EvaluationContext;
 import org.springframework.expression.Expression;
@@ -74,7 +77,10 @@ public class SimpleElasticsearchPersistentEntity<T> extends BasicPersistentEntit
 	private @Nullable VersionType versionType;
 	private boolean createIndexAndMapping;
 	private final Map<String, ElasticsearchPersistentProperty> fieldNamePropertyCache = new ConcurrentHashMap<>();
+
+	private EvaluationContextProvider evaluationContextProvider = EvaluationContextProvider.DEFAULT;;
 	private final ConcurrentHashMap<String, Expression> indexNameExpressions = new ConcurrentHashMap<>();
+	private final Lazy<EvaluationContext> indexNameEvaluationContext = Lazy.of(this::getIndexNameEvaluationContext);
 
 	public SimpleElasticsearchPersistentEntity(TypeInformation<T> typeInformation) {
 
@@ -302,12 +308,20 @@ public class SimpleElasticsearchPersistentEntity<T> extends BasicPersistentEntit
 		return seqNoPrimaryTermProperty;
 	}
 
+	@Nullable
 	@Override
 	public ElasticsearchPersistentProperty getJoinFieldProperty() {
 		return joinFieldProperty;
 	}
 
 	// region SpEL handling
+
+	@Override
+	public void setEvaluationContextProvider(EvaluationContextProvider provider) {
+		super.setEvaluationContextProvider(provider);
+		this.evaluationContextProvider = provider;
+	}
+
 	/**
 	 * resolves all the names in the IndexCoordinates object. If a name cannot be resolved, the original name is returned.
 	 *
@@ -316,14 +330,12 @@ public class SimpleElasticsearchPersistentEntity<T> extends BasicPersistentEntit
 	 */
 	private IndexCoordinates resolve(IndexCoordinates indexCoordinates) {
 
-		EvaluationContext context = getEvaluationContext(null);
-
 		String[] indexNames = indexCoordinates.getIndexNames();
 		String[] resolvedNames = new String[indexNames.length];
 
 		for (int i = 0; i < indexNames.length; i++) {
 			String indexName = indexNames[i];
-			resolvedNames[i] = resolve(context, indexName);
+			resolvedNames[i] = resolve(indexName);
 		}
 
 		return IndexCoordinates.of(resolvedNames);
@@ -332,22 +344,49 @@ public class SimpleElasticsearchPersistentEntity<T> extends BasicPersistentEntit
 	/**
 	 * tries to resolve the given name. If this is not successful, the original value is returned
 	 *
-	 * @param context SpEL evaluation context
 	 * @param name name to resolve
 	 * @return the resolved name or the input name if it cannot be resolved
 	 */
-	private String resolve(EvaluationContext context, String name) {
+	private String resolve(String name) {
 
 		Assert.notNull(name, "name must not be null");
 
-		Expression expression = indexNameExpressions.computeIfAbsent(name, s -> {
-			Expression expr = PARSER.parseExpression(name, ParserContext.TEMPLATE_EXPRESSION);
-			return expr instanceof LiteralExpression ? null : expr;
-		});
+		Expression expression = getExpressionForIndexName(name);
 
-		String resolvedName = expression != null ? expression.getValue(context, String.class) : null;
+		String resolvedName = expression != null ? expression.getValue(indexNameEvaluationContext.get(), String.class) : null;
 		return resolvedName != null ? resolvedName : name;
 	}
+
+	/**
+	 * returns an {@link Expression} for #name if name contains a {@link ParserContext#TEMPLATE_EXPRESSION} otherwise
+	 * returns {@literal null}.
+	 * 
+	 * @param name the name to get the expression for
+	 * @return Expression may be null
+	 */
+	@Nullable
+	private Expression getExpressionForIndexName(String name) {
+		return indexNameExpressions.computeIfAbsent(name, s -> {
+			Expression expr = PARSER.parseExpression(s, ParserContext.TEMPLATE_EXPRESSION);
+			return expr instanceof LiteralExpression ? null : expr;
+		});
+	}
+
+	/**
+	 * build the {@link EvaluationContext} considering {@link ExpressionDependencies} from the name returned by
+	 * {@link #getIndexName()}.
+	 * 
+	 * @return EvaluationContext
+	 */
+	private EvaluationContext getIndexNameEvaluationContext() {
+
+		Expression expression = getExpressionForIndexName(getIndexName());
+		ExpressionDependencies expressionDependencies = expression != null ? ExpressionDependencies.discover(expression)
+				: ExpressionDependencies.none();
+
+		return evaluationContextProvider.getEvaluationContext(null, expressionDependencies);
+	}
+
 	// endregion
 
 	@Override
@@ -363,5 +402,4 @@ public class SimpleElasticsearchPersistentEntity<T> extends BasicPersistentEntit
 				.put("index.refresh_interval", getRefreshInterval()).put("index.store.type", getIndexStoreType()).map();
 		return Document.from(map);
 	}
-
 }

src/main/java/org/springframework/data/elasticsearch/core/mapping/SimpleElasticsearchPersistentProperty.java

@@ -163,10 +163,13 @@ public class SimpleElasticsearchPersistentProperty extends
 				&& (isTemporalAccessor || isDate)) {
 			DateFormat dateFormat = field.format();
 
+			String property = getOwner().getType().getSimpleName() + "." + getName();
+
 			if (dateFormat == DateFormat.none) {
-				throw new MappingException(
-						String.format("Property %s is annotated with FieldType.%s but has no DateFormat defined",
-								getOwner().getType().getSimpleName() + "." + getName(), field.type().name()));
+				LOGGER.warn(
+						String.format("No DateFormat defined for property %s. Make sure you have a Converter registered for %s",
+								property, actualType.getSimpleName()));
+				return;
 			}
 
 			ElasticsearchDateConverter converter;
@@ -177,7 +180,7 @@ public class SimpleElasticsearchPersistentProperty extends
 				if (!StringUtils.hasLength(pattern)) {
 					throw new MappingException(
 							String.format("Property %s is annotated with FieldType.%s and a custom format but has no pattern defined",
-									getOwner().getType().getSimpleName() + "." + getName(), field.type().name()));
+									property, field.type().name()));
 				}
 
 				converter = ElasticsearchDateConverter.of(pattern);

src/main/java/org/springframework/data/elasticsearch/repository/query/ElasticsearchPartQuery.java

@@ -54,14 +54,14 @@ public class ElasticsearchPartQuery extends AbstractElasticsearchRepositoryQuery
 
 	public ElasticsearchPartQuery(ElasticsearchQueryMethod method, ElasticsearchOperations elasticsearchOperations) {
 		super(method, elasticsearchOperations);
-		this.tree = new PartTree(method.getName(), method.getEntityInformation().getJavaType());
+		this.tree = new PartTree(queryMethod.getName(), queryMethod.getResultProcessor().getReturnedType().getDomainType());
 		this.elasticsearchConverter = elasticsearchOperations.getElasticsearchConverter();
 		this.mappingContext = elasticsearchConverter.getMappingContext();
 	}
 
 	@Override
 	public Object execute(Object[] parameters) {
-		Class<?> clazz = queryMethod.getEntityInformation().getJavaType();
+		Class<?> clazz = queryMethod.getResultProcessor().getReturnedType().getDomainType();
 		ParametersParameterAccessor accessor = new ParametersParameterAccessor(queryMethod.getParameters(), parameters);
 
 		CriteriaQuery query = createQuery(accessor);

src/main/java/org/springframework/data/elasticsearch/repository/query/ElasticsearchStringQuery.java

@@ -71,7 +71,7 @@ public class ElasticsearchStringQuery extends AbstractElasticsearchRepositoryQue
 
 	@Override
 	public Object execute(Object[] parameters) {
-		Class<?> clazz = queryMethod.getEntityInformation().getJavaType();
+		Class<?> clazz = queryMethod.getResultProcessor().getReturnedType().getDomainType();
 		ParametersParameterAccessor accessor = new ParametersParameterAccessor(queryMethod.getParameters(), parameters);
 
 		StringQuery stringQuery = createQuery(accessor);

src/main/resources/changelog.txt

@@ -1,6 +1,68 @@
 Spring Data Elasticsearch Changelog
 ===================================
 
+Changes in version 4.1.2 (2020-12-09)
+-------------------------------------
+* DATAES-991 - Wrong value for TermVector(with_positions_offets_payloads).
+* DATAES-990 - Index creation fails with Authentication object cannot be null on startup.
+* DATAES-987 - IndexOperations getMapping fail when using index alias.
+* DATAES-978 - Accept DateFormat.none for a date property to enable custom Converters.
+* DATAES-977 - Fix versions in reference documentation for 4.1.
+* DATAES-973 - Release 4.1.2 (2020.0.2).
+* DATAES-972 - BeforeConvertCallback should be called before index query is built.
+* DATAES-543 - Adjust configuration support classes so they do not require proxying.
+
+
+Changes in version 4.2.0-M1 (2020-12-09)
+----------------------------------------
+* DATAES-995 - Code Cleanup after DATACMNS-1838.
+* DATAES-994 - Add setup for mutation testing.
+* DATAES-991 - Wrong value for TermVector(with_positions_offets_payloads).
+* DATAES-990 - Index creation fails with Authentication object cannot be null on startup.
+* DATAES-989 - Improve deprecation warning for id properties without annotation.
+* DATAES-988 - Allow specifying max results in NativeSearchQueryBuilder.
+* DATAES-987 - IndexOperations getMapping fail when using index alias.
+* DATAES-986 - Fix Javadoc.
+* DATAES-985 - Add builder method for track_total_hits to NativeSearchQueryBuilder.
+* DATAES-983 - Test dependency hoverfly-java-junit5 leaks into compile scope.
+* DATAES-978 - Accept DateFormat.none for a date property to enable custom Converters.
+* DATAES-976 - Implement CrudRepository.delete(Iterable<ID> ids).
+* DATAES-975 - Upgrade to Elasticsearch 7.10.
+* DATAES-974 - remove usage of deprecated WebClient exchange() method.
+* DATAES-972 - BeforeConvertCallback should be called before index query is built.
+* DATAES-971 - Fix tests for using a proxy with reactive client.
+* DATAES-970 - Take Testcontainers version from the Spring Data Build pom.
+* DATAES-969 - Use ResultProcessor in ElasticsearchPartQuery to build PartTree.
+* DATAES-968 - Enable Maven caching for Jenkins jobs.
+* DATAES-966 - Release 4.2 M1 (2021.0.0).
+* DATAES-882 - HLRC Configuration - add ability to set max connections for the underlying HttpClient.
+* DATAES-588 - Add support for custom callbacks in High Level/Low Level REST Client builder.
+* DATAES-543 - Adjust configuration support classes so they do not require proxying.
+* DATAES-362 - Add support for composable meta annotations.
+* DATAES-247 - Support OpType in IndexQuery.
+
+
+Changes in version 4.0.6.RELEASE (2020-12-09)
+---------------------------------------------
+* DATAES-991 - Wrong value for TermVector(with_positions_offets_payloads).
+* DATAES-969 - Use ResultProcessor in ElasticsearchPartQuery to build PartTree.
+* DATAES-968 - Enable Maven caching for Jenkins jobs.
+* DATAES-964 - Release 4.0.6 (Neumann SR6).
+
+
+Changes in version 3.2.12.RELEASE (2020-12-09)
+----------------------------------------------
+* DATAES-969 - Use ResultProcessor in ElasticsearchPartQuery to build PartTree.
+* DATAES-963 - Release 3.2.12 (Moore SR12).
+
+
+Changes in version 4.1.1 (2020-11-11)
+-------------------------------------
+* DATAES-969 - Use ResultProcessor in ElasticsearchPartQuery to build PartTree.
+* DATAES-968 - Enable Maven caching for Jenkins jobs.
+* DATAES-965 - Release 4.1.1 (2020.0.1).
+
+
 Changes in version 4.1.0 (2020-10-28)
 -------------------------------------
 * DATAES-962 - Deprecate Joda support.
@@ -1370,6 +1432,11 @@ Release Notes - Spring Data Elasticsearch - Version 1.0 M1 (2014-02-07)
 
 
 
+
+
+
+
+
 
 
 

src/main/resources/notice.txt

@@ -1,4 +1,4 @@
-Spring Data Elasticsearch 4.1 GA (2020.0.0)
+Spring Data Elasticsearch 4.1.2 (2020.0.2)
 Copyright (c) [2013-2019] Pivotal Software, Inc.
 
 This product is licensed to you under the Apache License, Version 2.0 (the "License").
@@ -20,3 +20,5 @@ conditions of the subcomponent's license, as noted in the LICENSE file.
 
 
 
+
+

src/test/java/org/springframework/data/elasticsearch/config/ElasticsearchConfigurationSupportUnitTests.java

@@ -18,6 +18,7 @@ package org.springframework.data.elasticsearch.config;
 import static org.assertj.core.api.Assertions.*;
 import static org.mockito.Mockito.*;
 
+import org.springframework.context.annotation.Bean;
 import reactor.core.publisher.Mono;
 
 import java.util.Collection;
@@ -114,6 +115,7 @@ public class ElasticsearchConfigurationSupportUnitTests {
 	static class ReactiveRestConfig extends AbstractReactiveElasticsearchConfiguration {
 
 		@Override
+		@Bean
 		public ReactiveElasticsearchClient reactiveElasticsearchClient() {
 			ReactiveElasticsearchClient client = mock(ReactiveElasticsearchClient.class);
 			when(client.info()).thenReturn(Mono

src/test/java/org/springframework/data/elasticsearch/core/ElasticsearchTemplateTests.java

@@ -121,14 +121,12 @@ import org.springframework.lang.Nullable;
  */
 public abstract class ElasticsearchTemplateTests {
 
+	protected static final String INDEX_NAME_JOIN_SAMPLE_ENTITY = "test-index-sample-join-template";
 	private static final String INDEX_NAME_SAMPLE_ENTITY = "test-index-sample-core-template";
 	private static final String INDEX_1_NAME = "test-index-1";
 	private static final String INDEX_2_NAME = "test-index-2";
 	private static final String INDEX_3_NAME = "test-index-3";
-
 	protected final IndexCoordinates index = IndexCoordinates.of(INDEX_NAME_SAMPLE_ENTITY);
-	protected static final String INDEX_NAME_JOIN_SAMPLE_ENTITY = "test-index-sample-join-template";
-
 	@Autowired protected ElasticsearchOperations operations;
 	protected IndexOperations indexOperations;
 
@@ -1466,6 +1464,30 @@ public abstract class ElasticsearchTemplateTests {
 		assertThat(mapping.get("properties")).isNotNull();
 	}
 
+	@Test // DATAES-987
+	@DisplayName("should read mappings from alias")
+	void shouldReadMappingsFromAlias() {
+
+		String aliasName = INDEX_NAME_SAMPLE_ENTITY + "alias";
+		indexOperations.alias( //
+				new AliasActions( //
+						new AliasAction.Add( //
+								AliasActionParameters.builder() //
+										.withIndices(INDEX_NAME_SAMPLE_ENTITY) //
+										.withAliases(aliasName) //
+										.build()) //
+				) //
+		);
+
+		IndexOperations aliasIndexOps = operations.indexOps(IndexCoordinates.of(aliasName));
+		Map<String, Object> mappingFromAlias = aliasIndexOps.getMapping();
+
+		assertThat(mappingFromAlias).isNotNull();
+		assertThat(
+				((Map<String, Object>) ((Map<String, Object>) mappingFromAlias.get("properties")).get("message")).get("type"))
+						.isEqualTo("text");
+	}
+
 	@Test
 	public void shouldDeleteIndexForGivenEntity() {
 

src/test/java/org/springframework/data/elasticsearch/core/event/ElasticsearchOperationsCallbackIntegrationTest.java

@@ -0,0 +1,235 @@
+/*
+ * Copyright 2020 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.springframework.data.elasticsearch.core.event;
+
+import static org.assertj.core.api.Assertions.*;
+import static org.mockito.Mockito.*;
+
+import lombok.Data;
+
+import java.util.Collections;
+import java.util.List;
+
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+import org.mockito.ArgumentCaptor;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.data.annotation.Id;
+import org.springframework.data.elasticsearch.annotations.Document;
+import org.springframework.data.elasticsearch.annotations.JoinTypeRelation;
+import org.springframework.data.elasticsearch.annotations.JoinTypeRelations;
+import org.springframework.data.elasticsearch.core.AbstractElasticsearchTemplate;
+import org.springframework.data.elasticsearch.core.ElasticsearchOperations;
+import org.springframework.data.elasticsearch.core.IndexOperations;
+import org.springframework.data.elasticsearch.core.join.JoinField;
+import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates;
+import org.springframework.data.elasticsearch.core.query.IndexQuery;
+import org.springframework.data.elasticsearch.core.query.SeqNoPrimaryTerm;
+import org.springframework.data.elasticsearch.junit.jupiter.SpringIntegrationTest;
+import org.springframework.lang.Nullable;
+import org.springframework.stereotype.Component;
+
+/**
+ * @author Peter-Josef Meisch
+ * @author Roman Puchkovskiy
+ */
+@SpringIntegrationTest
+abstract class ElasticsearchOperationsCallbackIntegrationTest {
+
+	private static final String INDEX = "test-operations-callback";
+	@Autowired private ElasticsearchOperations originalOperations;
+	// need a spy here on the abstract implementation class
+	private AbstractElasticsearchTemplate operations;
+
+	@Nullable private static SeqNoPrimaryTerm seqNoPrimaryTerm = null;
+
+	@Configuration
+	static class Config {
+
+		@Component
+		static class SampleEntityBeforeConvertCallback implements BeforeConvertCallback<SampleEntity> {
+			@Override
+			public SampleEntity onBeforeConvert(SampleEntity entity, IndexCoordinates index) {
+				entity.setText("converted");
+				JoinField<String> joinField = new JoinField<>("answer", "42");
+				entity.setJoinField(joinField);
+				if (seqNoPrimaryTerm != null) {
+					entity.setSeqNoPrimaryTerm(seqNoPrimaryTerm);
+				}
+				return entity;
+			}
+		}
+	}
+
+	@BeforeEach
+	void setUp() {
+		seqNoPrimaryTerm = null;
+		operations = (AbstractElasticsearchTemplate) spy(originalOperations);
+
+		IndexOperations indexOps = operations.indexOps(SampleEntity.class);
+		indexOps.delete();
+		indexOps.create();
+		indexOps.putMapping(SampleEntity.class);
+
+		// store one entity to have a seq_no and primary_term
+		final SampleEntity initial = new SampleEntity("1", "initial");
+		final SampleEntity saved = operations.save(initial);
+		seqNoPrimaryTerm = saved.getSeqNoPrimaryTerm();
+	}
+
+	@AfterEach
+	void tearDown() {
+		IndexOperations indexOps = operations.indexOps(SampleEntity.class);
+		indexOps.delete();
+	}
+
+	@Test // DATAES-68
+	void shouldCallBeforeConvertCallback() {
+		SampleEntity entity = new SampleEntity("1", "test");
+
+		SampleEntity saved = operations.save(entity);
+
+		assertThat(saved.getText()).isEqualTo("converted");
+	}
+
+	@Test // DATAES-972
+	@DisplayName("should apply conversion result to IndexQuery on save")
+	void shouldApplyConversionResultToIndexQueryOnSave() {
+
+		SampleEntity entity = new SampleEntity("1", "test");
+
+		operations.save(entity);
+
+		ArgumentCaptor<IndexQuery> indexQueryCaptor = ArgumentCaptor.forClass(IndexQuery.class);
+		verify(operations, times(2)).doIndex(indexQueryCaptor.capture(), any());
+
+		final IndexQuery capturedIndexQuery = indexQueryCaptor.getValue();
+		SampleEntity convertedEntity = (SampleEntity) capturedIndexQuery.getObject();
+
+		final JoinField<String> joinField = convertedEntity.getJoinField();
+		assertThat(joinField.getName()).isEqualTo("answer");
+		assertThat(joinField.getParent()).isEqualTo("42");
+		assertThat(capturedIndexQuery.getRouting()).isEqualTo("42");
+		assertThat(capturedIndexQuery.getSeqNo()).isEqualTo(seqNoPrimaryTerm.getSequenceNumber());
+		assertThat(capturedIndexQuery.getPrimaryTerm()).isEqualTo(seqNoPrimaryTerm.getPrimaryTerm());
+	}
+
+	@Test // DATAES-972
+	@DisplayName("should apply conversion result to IndexQuery when not set ")
+	void shouldApplyConversionResultToIndexQueryWhenNotSet() {
+
+		SampleEntity entity = new SampleEntity("1", "test");
+
+		final IndexQuery indexQuery = new IndexQuery();
+		indexQuery.setId(entity.getId());
+		indexQuery.setObject(entity);
+
+		operations.index(indexQuery, IndexCoordinates.of(INDEX));
+
+		ArgumentCaptor<IndexQuery> indexQueryCaptor = ArgumentCaptor.forClass(IndexQuery.class);
+		verify(operations, times(2)).doIndex(indexQueryCaptor.capture(), any());
+
+		final IndexQuery capturedIndexQuery = indexQueryCaptor.getValue();
+		SampleEntity convertedEntity = (SampleEntity) capturedIndexQuery.getObject();
+
+		final JoinField<String> joinField = convertedEntity.getJoinField();
+		assertThat(joinField.getName()).isEqualTo("answer");
+		assertThat(joinField.getParent()).isEqualTo("42");
+		assertThat(capturedIndexQuery.getRouting()).isEqualTo("42");
+		assertThat(capturedIndexQuery.getSeqNo()).isEqualTo(seqNoPrimaryTerm.getSequenceNumber());
+		assertThat(capturedIndexQuery.getPrimaryTerm()).isEqualTo(seqNoPrimaryTerm.getPrimaryTerm());
+	}
+
+	@Test // DATAES-972
+	@DisplayName("should not apply conversion result to IndexQuery when already set ")
+	void shouldNotApplyConversionResultToIndexQueryWhenAlreadySet() {
+
+		SeqNoPrimaryTerm seqNoPrimaryTermOriginal = seqNoPrimaryTerm;
+		seqNoPrimaryTerm = new SeqNoPrimaryTerm(7, 8);
+
+		SampleEntity entity = new SampleEntity("1", "test");
+
+		final IndexQuery indexQuery = new IndexQuery();
+		indexQuery.setId(entity.getId());
+		indexQuery.setObject(entity);
+		indexQuery.setRouting("12");
+		indexQuery.setSeqNo(seqNoPrimaryTermOriginal.getSequenceNumber());
+		indexQuery.setPrimaryTerm(seqNoPrimaryTermOriginal.getPrimaryTerm());
+
+		operations.index(indexQuery, IndexCoordinates.of(INDEX));
+
+		ArgumentCaptor<IndexQuery> indexQueryCaptor = ArgumentCaptor.forClass(IndexQuery.class);
+		verify(operations, times(2)).doIndex(indexQueryCaptor.capture(), any());
+
+		final IndexQuery capturedIndexQuery = indexQueryCaptor.getValue();
+		SampleEntity convertedEntity = (SampleEntity) capturedIndexQuery.getObject();
+
+		final JoinField<String> joinField = convertedEntity.getJoinField();
+		assertThat(joinField.getName()).isEqualTo("answer");
+		assertThat(joinField.getParent()).isEqualTo("42");
+		assertThat(capturedIndexQuery.getRouting()).isEqualTo("12");
+		assertThat(capturedIndexQuery.getSeqNo()).isEqualTo(seqNoPrimaryTermOriginal.getSequenceNumber());
+		assertThat(capturedIndexQuery.getPrimaryTerm()).isEqualTo(seqNoPrimaryTermOriginal.getPrimaryTerm());
+	}
+
+	@Test // DATAES-972
+	@DisplayName("should apply conversion result to IndexQuery in bulkIndex")
+	void shouldApplyConversionResultToIndexQueryInBulkIndex() {
+
+		SampleEntity entity = new SampleEntity("1", "test");
+
+		final IndexQuery indexQuery = new IndexQuery();
+		indexQuery.setId(entity.getId());
+		indexQuery.setObject(entity);
+
+		operations.bulkIndex(Collections.singletonList(indexQuery), SampleEntity.class);
+
+		ArgumentCaptor<List<IndexQuery>> indexQueryListCaptor = ArgumentCaptor.forClass(List.class);
+		verify(operations, times(1)).bulkOperation(indexQueryListCaptor.capture(), any(), any());
+
+		final List<IndexQuery> capturedIndexQueries = indexQueryListCaptor.getValue();
+		assertThat(capturedIndexQueries).hasSize(1);
+		final IndexQuery capturedIndexQuery = capturedIndexQueries.get(0);
+		SampleEntity convertedEntity = (SampleEntity) capturedIndexQuery.getObject();
+
+		final JoinField<String> joinField = convertedEntity.getJoinField();
+		assertThat(joinField.getName()).isEqualTo("answer");
+		assertThat(joinField.getParent()).isEqualTo("42");
+		assertThat(capturedIndexQuery.getRouting()).isEqualTo("42");
+		assertThat(capturedIndexQuery.getSeqNo()).isEqualTo(seqNoPrimaryTerm.getSequenceNumber());
+		assertThat(capturedIndexQuery.getPrimaryTerm()).isEqualTo(seqNoPrimaryTerm.getPrimaryTerm());
+	}
+
+	@Data
+	@Document(indexName = INDEX)
+	static class SampleEntity {
+		@Id private String id;
+		private String text;
+
+		@JoinTypeRelations(relations = { @JoinTypeRelation(parent = "question",
+				children = { "answer" }) }) @Nullable private JoinField<String> joinField;
+
+		private SeqNoPrimaryTerm seqNoPrimaryTerm;
+
+		public SampleEntity(String id, String text) {
+			this.id = id;
+			this.text = text;
+		}
+	}
+}

src/test/java/org/springframework/data/elasticsearch/core/event/ElasticsearchOperationsCallbackTest.java

@@ -1,102 +0,0 @@
-/*
- * Copyright 2020 the original author or authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *      https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.springframework.data.elasticsearch.core.event;
-
-import static org.assertj.core.api.Assertions.*;
-
-import org.junit.jupiter.api.AfterEach;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.context.annotation.Configuration;
-import org.springframework.data.annotation.Id;
-import org.springframework.data.elasticsearch.annotations.Document;
-import org.springframework.data.elasticsearch.core.ElasticsearchOperations;
-import org.springframework.data.elasticsearch.core.IndexOperations;
-import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates;
-import org.springframework.stereotype.Component;
-
-/**
- * @author Peter-Josef Meisch
- * @author Roman Puchkovskiy
- */
-abstract class ElasticsearchOperationsCallbackTest {
-
-	@Autowired private ElasticsearchOperations operations;
-
-	@Configuration
-	static class Config {
-
-		@Component
-		static class SampleEntityBeforeConvertCallback implements BeforeConvertCallback<SampleEntity> {
-			@Override
-			public SampleEntity onBeforeConvert(SampleEntity entity, IndexCoordinates index) {
-				entity.setText("converted");
-				return entity;
-			}
-		}
-	}
-
-	@BeforeEach
-	void setUp() {
-		IndexOperations indexOps = operations.indexOps(SampleEntity.class);
-		indexOps.delete();
-		indexOps.create();
-		indexOps.putMapping(SampleEntity.class);
-	}
-
-	@AfterEach
-	void tearDown() {
-		IndexOperations indexOps = operations.indexOps(SampleEntity.class);
-		indexOps.delete();
-	}
-
-	@Test
-	void shouldCallBeforeConvertCallback() {
-		SampleEntity entity = new SampleEntity("1", "test");
-
-		SampleEntity saved = operations.save(entity);
-
-		assertThat(saved.getText()).isEqualTo("converted");
-	}
-
-	@Document(indexName = "test-operations-callback")
-	static class SampleEntity {
-		@Id private String id;
-		private String text;
-
-		public SampleEntity(String id, String text) {
-			this.id = id;
-			this.text = text;
-		}
-
-		public String getId() {
-			return id;
-		}
-
-		public void setId(String id) {
-			this.id = id;
-		}
-
-		public String getText() {
-			return text;
-		}
-
-		public void setText(String text) {
-			this.text = text;
-		}
-	}
-}

src/test/java/org/springframework/data/elasticsearch/core/event/ElasticsearchRestOperationsCallbackIntegrationTest.java

@@ -16,12 +16,11 @@
 package org.springframework.data.elasticsearch.core.event;
 
 import org.springframework.data.elasticsearch.junit.jupiter.ElasticsearchRestTemplateConfiguration;
-import org.springframework.data.elasticsearch.junit.jupiter.SpringIntegrationTest;
 import org.springframework.test.context.ContextConfiguration;
 
 /**
  * @author Peter-Josef Meisch
  */
-@SpringIntegrationTest
-@ContextConfiguration(classes = { ElasticsearchRestTemplateConfiguration.class, ElasticsearchOperationsCallbackTest.Config.class })
-class ElasticsearchRestOperationsCallbackTest extends ElasticsearchOperationsCallbackTest {}
+@ContextConfiguration(classes = { ElasticsearchRestTemplateConfiguration.class,
+		ElasticsearchOperationsCallbackIntegrationTest.Config.class })
+class ElasticsearchRestOperationsCallbackIntegrationTest extends ElasticsearchOperationsCallbackIntegrationTest {}

src/test/java/org/springframework/data/elasticsearch/core/event/ElasticsearchTransportOperationsCallbackIntegrationTest.java

@@ -16,12 +16,10 @@
 package org.springframework.data.elasticsearch.core.event;
 
 import org.springframework.data.elasticsearch.junit.jupiter.ElasticsearchTemplateConfiguration;
-import org.springframework.data.elasticsearch.junit.jupiter.SpringIntegrationTest;
 import org.springframework.test.context.ContextConfiguration;
 
 /**
  * @author Peter-Josef Meisch
  */
-@SpringIntegrationTest
-@ContextConfiguration(classes = { ElasticsearchTemplateConfiguration.class, ElasticsearchOperationsCallbackTest.Config.class })
-class ElasticsearchTransportOperationsCallbackTest extends ElasticsearchOperationsCallbackTest {}
+@ContextConfiguration(classes = { ElasticsearchTemplateConfiguration.class, ElasticsearchOperationsCallbackIntegrationTest.Config.class })
+class ElasticsearchTransportOperationsCallbackIntegrationTest extends ElasticsearchOperationsCallbackIntegrationTest {}

src/test/java/org/springframework/data/elasticsearch/core/index/MappingBuilderIntegrationTests.java

@@ -45,6 +45,7 @@ import org.assertj.core.data.Percentage;
 import org.elasticsearch.search.suggest.completion.context.ContextMapping;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.DisplayName;
 import org.junit.jupiter.api.Test;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.data.annotation.Id;
@@ -249,6 +250,16 @@ public class MappingBuilderIntegrationTests extends MappingContextBaseTests {
 		assertThat(fieldLastName.get("copy_to")).isEqualTo(copyToValue);
 	}
 
+	@Test // DATAES-991
+	@DisplayName("should write correct TermVector values")
+	void shouldWriteCorrectTermVectorValues() {
+
+		IndexOperations indexOps = operations.indexOps(TermVectorFieldEntity.class);
+		indexOps.create();
+		indexOps.putMapping();
+
+	}
+
 	/**
 	 * @author Xiao Yu
 	 */
@@ -621,4 +632,19 @@ public class MappingBuilderIntegrationTests extends MappingContextBaseTests {
 
 		@Field(type = FieldType.Rank_Features) private Map<String, Integer> topics;
 	}
+
+	@Data
+	@Document(indexName = "termvectors-test")
+	static class TermVectorFieldEntity {
+		@Id private String id;
+		@Field(type = FieldType.Text, termVector = TermVector.no) private String no;
+		@Field(type = FieldType.Text, termVector = TermVector.yes) private String yes;
+		@Field(type = FieldType.Text, termVector = TermVector.with_positions) private String with_positions;
+		@Field(type = FieldType.Text, termVector = TermVector.with_offsets) private String with_offsets;
+		@Field(type = FieldType.Text, termVector = TermVector.with_positions_offsets) private String with_positions_offsets;
+		@Field(type = FieldType.Text,
+				termVector = TermVector.with_positions_payloads) private String with_positions_payloads;
+		@Field(type = FieldType.Text,
+				termVector = TermVector.with_positions_offsets_payloads) private String with_positions_offsets_payloads;
+	}
 }

src/test/java/org/springframework/data/elasticsearch/core/index/MappingContextBaseTests.java

@@ -34,8 +34,9 @@ abstract class MappingContextBaseTests {
 
 	private SimpleElasticsearchMappingContext setupMappingContext() {
 
-		SimpleElasticsearchMappingContext mappingContext = new ElasticsearchConfigurationSupport()
-				.elasticsearchMappingContext();
+		ElasticsearchConfigurationSupport configurationSupport = new ElasticsearchConfigurationSupport();
+		SimpleElasticsearchMappingContext mappingContext = configurationSupport
+				.elasticsearchMappingContext(configurationSupport.elasticsearchCustomConversions());
 		mappingContext.initialize();
 		return mappingContext;
 	}

src/test/java/org/springframework/data/elasticsearch/core/mapping/SimpleElasticsearchPersistentPropertyUnitTests.java

@@ -192,20 +192,6 @@ public class SimpleElasticsearchPersistentPropertyUnitTests {
 		assertThat(seqNoProperty.isReadable()).isFalse();
 	}
 
-	@Test // DATAES-828
-	void shouldRequireFormatForDateField() {
-		assertThatExceptionOfType(MappingException.class) //
-				.isThrownBy(() -> context.getRequiredPersistentEntity(DateFieldWithNoFormat.class)) //
-				.withMessageContaining("date");
-	}
-
-	@Test // DATAES-828
-	void shouldRequireFormatForDateNanosField() {
-		assertThatExceptionOfType(MappingException.class) //
-				.isThrownBy(() -> context.getRequiredPersistentEntity(DateNanosFieldWithNoFormat.class)) //
-				.withMessageContaining("date");
-	}
-
 	@Test // DATAES-924
 	@DisplayName("should require pattern for custom date format")
 	void shouldRequirePatternForCustomDateFormat() {

src/test/java/org/springframework/data/elasticsearch/junit/jupiter/ElasticsearchRestTemplateConfiguration.java

@@ -68,9 +68,9 @@ public class ElasticsearchRestTemplateConfiguration extends AbstractElasticsearc
 	}
 
 	@Override
-	public ElasticsearchOperations elasticsearchOperations(ElasticsearchConverter elasticsearchConverter) {
-		RestHighLevelClient client = elasticsearchClient();
-		return new ElasticsearchRestTemplate(client, elasticsearchConverter) {
+	public ElasticsearchOperations elasticsearchOperations(ElasticsearchConverter elasticsearchConverter,
+			RestHighLevelClient elasticsearchClient) {
+		return new ElasticsearchRestTemplate(elasticsearchClient, elasticsearchConverter) {
 			@Override
 			public <T> T execute(ClientCallback<T> callback) {
 				try {