Release version 4.0.8 (Neumann SR8).

See #1697

.travis.yml

@@ -1,7 +0,0 @@
-dist: xenial
-
-language: java
-
-sudo: true
-
-script: "./mvnw -Pjava11 clean dependency:list test -Dsort -U -B"

CI.adoc

@@ -9,7 +9,7 @@ image:https://jenkins.spring.io/buildStatus/icon?job=spring-data-elasticsearch%2
 Since this pipeline is purely Docker-based, it's easy to:
 
 * Debug what went wrong on your local machine.
-* Test out a a tweak to your `test.sh` script before sending it out.
+* Test out a a tweak to your `verify.sh` script before sending it out.
 * Experiment against a new image before submitting your pull request.
 
 All of these use cases are great reasons to essentially run what the CI server does on your local machine.

Jenkinsfile

@@ -3,7 +3,7 @@ pipeline {
 
 	triggers {
 		pollSCM 'H/10 * * * *'
-		upstream(upstreamProjects: "spring-data-commons/master", threshold: hudson.model.Result.SUCCESS)
+		upstream(upstreamProjects: "spring-data-commons/2.3.x", threshold: hudson.model.Result.SUCCESS)
 	}
 
 	options {
@@ -15,59 +15,83 @@ pipeline {
 		stage("test: baseline (jdk8)") {
 			when {
 				anyOf {
-					branch 'master'
+					branch '4.0.x'
 					not { triggeredBy 'UpstreamCause' }
 				}
 			}
 			agent {
-				docker {
-					image 'adoptopenjdk/openjdk8:latest'
-					label 'data'
-					args '-v $HOME:/tmp/jenkins-home'
-				}
+				label 'data'
 			}
 			options { timeout(time: 30, unit: 'MINUTES') }
+
+			environment {
+				DOCKER_HUB = credentials('hub.docker.com-springbuildmaster')
+			}
+
 			steps {
-				sh 'rm -rf ?'
-				sh 'MAVEN_OPTS="-Duser.name=jenkins -Duser.home=/tmp/jenkins-home" ./mvnw clean dependency:list test -Dsort -U -B'
+				script {
+					docker.withRegistry('', 'hub.docker.com-springbuildmaster') {
+						docker.image('adoptopenjdk/openjdk8:latest').inside('-u root -v /var/run/docker.sock:/var/run/docker.sock -v /usr/bin/docker:/usr/bin/docker -v $HOME:/tmp/jenkins-home') {
+							sh "docker login --username ${DOCKER_HUB_USR} --password ${DOCKER_HUB_PSW}"
+							sh 'PROFILE=none ci/verify.sh'
+							sh "ci/clean.sh"
+						}
+					}
+				}
 			}
 		}
 
 		stage("Test other configurations") {
 			when {
 				anyOf {
-					branch 'master'
+					branch '4.0.x'
 					not { triggeredBy 'UpstreamCause' }
 				}
 			}
 			parallel {
 				stage("test: baseline (jdk11)") {
 					agent {
-						docker {
-							image 'adoptopenjdk/openjdk11:latest'
-							label 'data'
-							args '-v $HOME:/tmp/jenkins-home'
-						}
+						label 'data'
 					}
 					options { timeout(time: 30, unit: 'MINUTES') }
+
+					environment {
+						DOCKER_HUB = credentials('hub.docker.com-springbuildmaster')
+					}
+
 					steps {
-						sh 'rm -rf ?'
-						sh 'MAVEN_OPTS="-Duser.name=jenkins -Duser.home=/tmp/jenkins-home" ./mvnw -Pjava11 clean dependency:list test -Dsort -U -B'
+						script {
+							docker.withRegistry('', 'hub.docker.com-springbuildmaster') {
+								docker.image('adoptopenjdk/openjdk11:latest').inside('-u root -v /var/run/docker.sock:/var/run/docker.sock -v /usr/bin/docker:/usr/bin/docker -v $HOME:/tmp/jenkins-home') {
+									sh "docker login --username ${DOCKER_HUB_USR} --password ${DOCKER_HUB_PSW}"
+									sh 'PROFILE=java11 ci/verify.sh'
+									sh "ci/clean.sh"
+								}
+							}
+						}
 					}
 				}
 
 				stage("test: baseline (jdk12)") {
 					agent {
-						docker {
-							image 'adoptopenjdk/openjdk12:latest'
-							label 'data'
-							args '-v $HOME:/tmp/jenkins-home'
-						}
+						label 'data'
 					}
 					options { timeout(time: 30, unit: 'MINUTES') }
+
+					environment {
+						DOCKER_HUB = credentials('hub.docker.com-springbuildmaster')
+					}
+
 					steps {
-						sh 'rm -rf ?'
-						sh 'MAVEN_OPTS="-Duser.name=jenkins -Duser.home=/tmp/jenkins-home" ./mvnw -Pjava11 clean dependency:list test -Dsort -U -B'
+						script {
+							docker.withRegistry('', 'hub.docker.com-springbuildmaster') {
+								docker.image('adoptopenjdk/openjdk12:latest').inside('-u root -v /var/run/docker.sock:/var/run/docker.sock -v /usr/bin/docker:/usr/bin/docker -v $HOME:/tmp/jenkins-home') {
+									sh "docker login --username ${DOCKER_HUB_USR} --password ${DOCKER_HUB_PSW}"
+									sh 'PROFILE=java11 ci/verify.sh'
+									sh "ci/clean.sh"
+								}
+							}
+						}
 					}
 				}
 			}
@@ -76,16 +100,12 @@ pipeline {
 		stage('Release to artifactory') {
 			when {
 				anyOf {
-					branch 'master'
+					branch '4.0.x'
 					not { triggeredBy 'UpstreamCause' }
 				}
 			}
 			agent {
-				docker {
-					image 'adoptopenjdk/openjdk8:latest'
-					label 'data'
-					args '-v $HOME:/tmp/jenkins-home'
-				}
+				label 'data'
 			}
 			options { timeout(time: 20, unit: 'MINUTES') }
 
@@ -94,27 +114,28 @@ pipeline {
 			}
 
 			steps {
-				sh 'rm -rf ?'
-				sh 'MAVEN_OPTS="-Duser.name=jenkins -Duser.home=/tmp/jenkins-home" ./mvnw -Pci,artifactory ' +
-						'-Dartifactory.server=https://repo.spring.io ' +
-						"-Dartifactory.username=${ARTIFACTORY_USR} " +
-						"-Dartifactory.password=${ARTIFACTORY_PSW} " +
-						"-Dartifactory.staging-repository=libs-snapshot-local " +
-						"-Dartifactory.build-name=spring-data-elasticsearch " +
-						"-Dartifactory.build-number=${BUILD_NUMBER} " +
-						'-Dmaven.test.skip=true clean deploy -U -B'
+				script {
+					docker.withRegistry('', 'hub.docker.com-springbuildmaster') {
+						docker.image('adoptopenjdk/openjdk8:latest').inside('-v $HOME:/tmp/jenkins-home') {
+							sh 'MAVEN_OPTS="-Duser.name=jenkins -Duser.home=/tmp/jenkins-home" ./mvnw -Pci,artifactory -Dmaven.repo.local=/tmp/jenkins-home/.m2/spring-data-elasticsearch-non-root ' +
+									'-Dartifactory.server=https://repo.spring.io ' +
+									"-Dartifactory.username=${ARTIFACTORY_USR} " +
+									"-Dartifactory.password=${ARTIFACTORY_PSW} " +
+									"-Dartifactory.staging-repository=libs-snapshot-local " +
+									"-Dartifactory.build-name=spring-data-elasticsearch " +
+									"-Dartifactory.build-number=${BUILD_NUMBER} " +
+									'-Dmaven.test.skip=true clean deploy -U -B'
+						}
+					}
+				}
 			}
 		}
 		stage('Publish documentation') {
 			when {
-				branch 'master'
+				branch '4.0.x'
 			}
 			agent {
-				docker {
-					image 'adoptopenjdk/openjdk8:latest'
-					label 'data'
-					args '-v $HOME:/tmp/jenkins-home'
-				}
+				label 'data'
 			}
 			options { timeout(time: 20, unit: 'MINUTES') }
 
@@ -123,12 +144,18 @@ pipeline {
 			}
 
 			steps {
-				sh 'MAVEN_OPTS="-Duser.name=jenkins -Duser.home=/tmp/jenkins-home" ./mvnw -Pci,distribute ' +
-						'-Dartifactory.server=https://repo.spring.io ' +
-						"-Dartifactory.username=${ARTIFACTORY_USR} " +
-						"-Dartifactory.password=${ARTIFACTORY_PSW} " +
-						"-Dartifactory.distribution-repository=temp-private-local " +
-						'-Dmaven.test.skip=true clean deploy -U -B'
+				script {
+					docker.withRegistry('', 'hub.docker.com-springbuildmaster') {
+						docker.image('adoptopenjdk/openjdk8:latest').inside('-v $HOME:/tmp/jenkins-home') {
+							sh 'MAVEN_OPTS="-Duser.name=jenkins -Duser.home=/tmp/jenkins-home" ./mvnw -Pci,distribute -Dmaven.repo.local=/tmp/jenkins-home/.m2/spring-data-elasticsearch-non-root ' +
+									'-Dartifactory.server=https://repo.spring.io ' +
+									"-Dartifactory.username=${ARTIFACTORY_USR} " +
+									"-Dartifactory.password=${ARTIFACTORY_PSW} " +
+									"-Dartifactory.distribution-repository=temp-private-local " +
+									'-Dmaven.test.skip=true clean deploy -U -B'
+						}
+					}
+				}
 			}
 		}
 	}

ci/clean.sh

@@ -0,0 +1,6 @@
+#!/bin/bash -x
+
+set -euo pipefail
+
+MAVEN_OPTS="-Duser.name=jenkins -Duser.home=/tmp/jenkins-home" \
+  ./mvnw clean -Dmaven.repo.local=/tmp/jenkins-home/.m2/spring-data-elasticsearch

ci/verify.sh

@@ -0,0 +1,10 @@
+#!/bin/bash -x
+
+set -euo pipefail
+
+mkdir -p /tmp/jenkins-home/.m2/spring-data-elasticsearch
+chown -R 1001:1001 .
+
+MAVEN_OPTS="-Duser.name=jenkins -Duser.home=/tmp/jenkins-home" \
+  ./mvnw \
+  -P${PROFILE} clean dependency:list verify -Dsort -U -B -Dmaven.repo.local=/tmp/jenkins-home/.m2/spring-data-elasticsearch

pom.xml

@@ -5,12 +5,12 @@
 
 	<groupId>org.springframework.data</groupId>
 	<artifactId>spring-data-elasticsearch</artifactId>
-	<version>4.0.0.M4</version>
+	<version>4.0.8.RELEASE</version>
 
 	<parent>
 		<groupId>org.springframework.data.build</groupId>
 		<artifactId>spring-data-parent</artifactId>
-		<version>2.3.0.M4</version>
+		<version>2.3.8.RELEASE</version>
 	</parent>
 
 	<name>Spring Data Elasticsearch</name>
@@ -19,9 +19,9 @@
 
 	<properties>
 		<commonslang>2.6</commonslang>
-		<elasticsearch>7.6.1</elasticsearch>
+		<elasticsearch>7.6.2</elasticsearch>
 		<log4j>2.9.1</log4j>
-		<springdata.commons>2.3.0.M4</springdata.commons>
+		<springdata.commons>2.3.8.RELEASE</springdata.commons>
 		<netty>4.1.39.Final</netty>
 		<java-module-name>spring.data.elasticsearch</java-module-name>
 	</properties>
@@ -252,7 +252,7 @@
 		<dependency>
 			<groupId>com.github.tomakehurst</groupId>
 			<artifactId>wiremock-jre8</artifactId>
-			<version>2.25.1</version>
+			<version>2.26.3</version>
 			<scope>test</scope>
 			<exclusions>
 				<!-- these exclusions are needed because of Elasticsearch JarHell-->
@@ -292,6 +292,23 @@
 	</dependencies>
 
 	<build>
+		<resources>
+			<resource>
+				<directory>src/main/resources</directory>
+				<filtering>true</filtering>
+				<includes>
+					<include>**/versions.properties</include>
+				</includes>
+			</resource>
+			<resource>
+				<directory>src/main/resources</directory>
+				<filtering>false</filtering>
+				<excludes>
+					<exclude>**/versions.properties</exclude>
+				</excludes>
+			</resource>
+		</resources>
+
 		<plugins>
 			<!--
 			please do not remove this configuration for surefire - we need that to avoid issue with jar hell
@@ -304,6 +321,7 @@
 					<useFile>false</useFile>
 					<includes>
 						<include>**/*Tests.java</include>
+						<include>**/*Test.java</include>
 					</includes>
 					<systemPropertyVariables>
 						<es.set.netty.runtime.available.processors>false</es.set.netty.runtime.available.processors>
@@ -355,8 +373,8 @@
 
 	<repositories>
 		<repository>
-			<id>spring-libs-milestone</id>
-			<url>https://repo.spring.io/libs-milestone</url>
+			<id>spring-libs-release</id>
+			<url>https://repo.spring.io/libs-release</url>
 		</repository>
 	</repositories>
 
@@ -365,6 +383,11 @@
 			<id>spring-plugins-release</id>
 			<url>https://repo.spring.io/plugins-release</url>
 		</pluginRepository>
+		<pluginRepository>
+			<id>bintray-plugins</id>
+			<name>bintray-plugins</name>
+			<url>https://jcenter.bintray.com</url>
+		</pluginRepository>
 	</pluginRepositories>
 
 </project>

src/main/asciidoc/index.adoc

@@ -5,13 +5,14 @@ BioMed Central Development Team; Oliver Drotbohm; Greg Turnquist; Christoph Stro
 ifdef::backend-epub3[:front-cover-image: image:epub-cover.png[Front Cover,1050,1600]]
 :spring-data-commons-docs: ../../../../spring-data-commons/src/main/asciidoc
 
-(C) 2013-2019 The original author(s).
+(C) 2013-2020 The original author(s).
 
 NOTE: Copies of this document may be made for your own use and for distribution to others, provided that you do not charge any fee for such copies and further provided that each copy contains this Copyright Notice, whether distributed in print or electronically.
 
 toc::[]
 
 include::preface.adoc[]
+
 :leveloffset: +1
 include::{spring-data-commons-docs}/repositories.adoc[]
 :leveloffset: -1
@@ -23,7 +24,15 @@ include::{spring-data-commons-docs}/repositories.adoc[]
 include::reference/elasticsearch-clients.adoc[]
 include::reference/elasticsearch-object-mapping.adoc[]
 include::reference/elasticsearch-operations.adoc[]
+
 include::reference/elasticsearch-repositories.adoc[]
+
+include::{spring-data-commons-docs}/auditing.adoc[]
+include::reference/elasticsearch-auditing.adoc[]
+
+include::{spring-data-commons-docs}/entity-callbacks.adoc[]
+include::reference/elasticsearch-entity-callbacks.adoc[leveloffset=+1]
+
 include::reference/elasticsearch-misc.adoc[]
 :leveloffset: -1
 
@@ -35,4 +44,5 @@ include::{spring-data-commons-docs}/repository-namespace-reference.adoc[]
 include::{spring-data-commons-docs}/repository-populator-namespace-reference.adoc[]
 include::{spring-data-commons-docs}/repository-query-keywords-reference.adoc[]
 include::{spring-data-commons-docs}/repository-query-return-types-reference.adoc[]
+include::reference/migration-guides.adoc[]
 :leveloffset: -1

src/main/asciidoc/preface.adoc

@@ -35,7 +35,7 @@ The following table shows the Elasticsearch versions that are used by Spring Dat
 [cols="^,^,^,^",options="header"]
 |===
 | Spring Data Release Train |Spring Data Elasticsearch |Elasticsearch | Spring Boot
-| Neumannfootnote:cdv[Currently in development] |4.0.xfootnote:cdv[]|7.6.1 |2.3.xfootnote:cdv[]
+| Neumannfootnote:cdv[Currently in development] |4.0.xfootnote:cdv[]|7.6.2 |2.3.xfootnote:cdv[]
 | Moore | 3.2.x |6.8.6 | 2.2.x
 | Lovelace | 3.1.x | 6.2.2 |2.1.x
 | Kayfootnote:oom[Out of maintenance] | 3.0.xfootnote:oom[] | 5.5.0 | 2.0.xfootnote:oom[]

src/main/asciidoc/reference/elasticsearch-auditing.adoc

@@ -0,0 +1,68 @@
+[[elasticsearch.auditing]]
+== Elasticsearch Auditing
+
+=== Preparing entities
+
+In order for the auditing code to be able to decide wether an entity instance is new, the entity must implement the `Persistable<ID>` interface which is defined as follows:
+
+[source,java]
+----
+package org.springframework.data.domain;
+
+import org.springframework.lang.Nullable;
+
+public interface Persistable<ID> {
+    @Nullable
+    ID getId();
+
+    boolean isNew();
+}
+----
+
+As the existence of an Id is not a sufficient criterion to determine if an enitity is new in Elasticsearch, additional information is necessary. One way is to use the creation-relevant auditing fields for this decision:
+
+A `Person` entity might look as follows - omitting getter and setter methods for brevity:
+
+[source,java]
+----
+@Document(indexName = "person")
+public class Person implements Persistable<Long> {
+    @Id private Long id;
+    private String lastName;
+    private String firstName;
+    @Field(type = FieldType.Date, format = DateFormat.basic_date_time)
+    private Instant createdDate;
+    private String createdBy
+    @Field(type = FieldType.Date, format = DateFormat.basic_date_time)
+    private Instant lastModifiedDate;
+    private String lastModifiedBy;
+
+    public Long getId() {                                                 <1>
+        return id;
+    }
+
+    @Override
+    public boolean isNew() {
+        return id == null || (createdDate == null && createdBy == null);  <2>
+    }
+}
+----
+<1> the getter also is the required implementation from the interface
+<2> an object is new if it either has no `id` or none of fields containing creation attributes are set.
+
+=== Activating auditing
+
+After the entities have been set up and providing the `AuditorAware` the Auditing must be activated by setting the `@EnableElasticsearchAuditing` on a configuration class:
+
+[source,java]
+----
+@Configuration
+@EnableElasticsearchRepositories
+@EnableElasticsearchAuditing
+class MyConfiguration {
+   // configuration code
+}
+----
+
+If your code contains more than one `AuditorAware` bean for different types, you must provide the name of the bean to use as an argument to the `auditorAwareRef` parameter of the 
+ `@EnableElasticsearchAuditing` annotation. 

src/main/asciidoc/reference/elasticsearch-clients.adoc

@@ -143,40 +143,47 @@ NOTE: The ReactiveClient response, especially for search operations, is bound to
 [[elasticsearch.clients.configuration]]
 == Client Configuration
 
-Client behaviour can be changed via the `ClientConfiguration` that allows to set options for SSL, connect and socket timeouts.
+Client behaviour can be changed via the `ClientConfiguration` that allows to set options for SSL, connect and socket timeouts, headers and other parameters.
 
 .Client Configuration
 ====
 [source,java]
 ----
-// optional if Basic Auhtentication is needed
 HttpHeaders httpHeaders = new HttpHeaders();
-httpHeaders.add("es-security-runas-user", "some-user")                  <1>
+httpHeaders.add("some-header", "on every request")                      <1>
 
 ClientConfiguration clientConfiguration = ClientConfiguration.builder()
   .connectedTo("localhost:9200", "localhost:9291")                      <2>
-  .withProxy("localhost:8888")                                          <3>
-  .withPathPrefix("ela")                                                <4>
-  .withConnectTimeout(Duration.ofSeconds(5))                            <5>
-  .withSocketTimeout(Duration.ofSeconds(3))                             <6>
-  .useSsl()                                                             <7>
+  .usingSsl()                                                           <3>
+  .withProxy("localhost:8888")                                          <4>
+  .withPathPrefix("ela")                                                <5>
+  .withConnectTimeout(Duration.ofSeconds(5))                            <6>
+  .withSocketTimeout(Duration.ofSeconds(3))                             <7>
   .withDefaultHeaders(defaultHeaders)                                   <8>
   .withBasicAuth(username, password)                                    <9>
+  .withHeaders(() -> {                                                  <10>
+    HttpHeaders headers = new HttpHeaders();
+    headers.add("currentTime", LocalDateTime.now().format(DateTimeFormatter.ISO_LOCAL_DATE_TIME));
+    return headers;
+  })
   . // ... other options
   .build();
 
 ----
 <1> Define default headers, if they need to be customized
 <2> Use the builder to provide cluster addresses, set default `HttpHeaders` or enable SSL.
-<3> Optionally set a proxy footnote:notreactive[not yet implemented for the reactive client].
-<4> Optionally set a path prefix, mostly used when different clusters a behind some reverse proxy.
-<5> Set the connection timeout. Default is 10 sec.
-<6> Set the socket timeout. Default is 5 sec.
-<7> Optionally enable SSL.
+<3> Optionally enable SSL.
+<4> Optionally set a proxy.
+<5> Optionally set a path prefix, mostly used when different clusters a behind some reverse proxy.
+<6> Set the connection timeout. Default is 10 sec.
+<7> Set the socket timeout. Default is 5 sec.
 <8> Optionally set headers.
 <9> Add basic authentication.
+<10> A `Supplier<Header>` function can be specified which is called every time before a request is sent to Elasticsearch - here, as an example, the current time is written in a header.
 ====
 
+IMPORTANT: Adding a Header supplier as shown in above example allows to inject headers that may change over the time, like authentication JWT tokens. If this is used in the reactive setup, the supplier function *must not* block!
+
 [[elasticsearch.clients.logging]]
 == Client Logging
 

src/main/asciidoc/reference/elasticsearch-entity-callbacks.adoc

@@ -0,0 +1,35 @@
+[[elasticsearch.entity-callbacks]]
+= Elasticsearch EntityCallbacks
+
+Spring Data Elasticsearch uses the `EntityCallback` API internally for its auditing support and reacts on the following callbacks:
+
+.Supported Entity Callbacks
+[%header,cols="4"]
+|===
+| Callback
+| Method
+| Description
+| Order
+
+| Reactive/BeforeConvertCallback
+| `onBeforeConvert(T entity, IndexCoordinates index)`
+| Invoked before a domain object is converted to `org.springframework.data.elasticsearch.core.document.Document`. Can return the `entity` or a modified entity which then will be converted.
+| `Ordered.LOWEST_PRECEDENCE`
+
+| Reactive/AfterConvertCallback
+| `onAfterConvert(T entity, Document document, IndexCoordinates indexCoordinates)`
+| Invoked after a domain object is converted from `org.springframework.data.elasticsearch.core.document.Document` on reading result data from Elasticsearch.
+| `Ordered.LOWEST_PRECEDENCE`
+
+| Reactive/AuditingEntityCallback
+| `onBeforeConvert(Object entity, IndexCoordinates index)`
+| Marks an auditable entity _created_ or _modified_
+| 100
+
+| Reactive/AfterSaveCallback
+| `T onAfterSave(T entity, IndexCoordinates index)`
+| Invoked after a domain object is saved.
+| `Ordered.LOWEST_PRECEDENCE`
+
+|===
+

src/main/asciidoc/reference/elasticsearch-migration-guide-3.2-4.0.adoc

@@ -0,0 +1,133 @@
+[[elasticsearch-migration-guide-3.2-4.0]]
+= Upgrading from 3.2.x to 4.0.x
+
+This section describes breaking changes from version 3.2.x to 4.0.x and how removed features can be replaced by new introduced features.
+
+[[elasticsearch-migration-guide-3.2-4.0.jackson-removal]]
+== Removal of the used Jackson Mapper
+
+One of the changes in version 4.0.x is that Spring Data Elasticsearch does not use the Jackson Mapper anymore to map an entity to the JSON representation needed for Elasticsearch (see <<elasticsearch.mapping>>). In version 3.2.x the Jackson Mapper was the default that was used. It was possible to switch to the meta-model based converter (named `ElasticsearchEntityMapper`) by explicitly configuring it (<<elasticsearch.mapping.meta-model>>).
+
+In version 4.0.x the meta-model based converter is the only one that is available and does not need to be configured explicitly. If you had a custom configuration to enable the meta-model converter by providing a bean like this:
+
+[source,java]
+----
+@Bean
+@Override
+public EntityMapper entityMapper() {
+
+  ElasticsearchEntityMapper entityMapper = new ElasticsearchEntityMapper(
+    elasticsearchMappingContext(), new DefaultConversionService()
+  );
+  entityMapper.setConversions(elasticsearchCustomConversions());
+
+  return entityMapper;
+}
+----
+
+You now have to remove this bean, the `ElasticsearchEntityMapper` interface has been removed.
+
+.Entity configuration
+Some users had custom Jackson annotations on the entity class, for example in order to define a custom name for the mapped document in Elasticsearch or to configure date conversions. These are not taken into account anymore. The needed functionality is now provided with Spring Data Elasticsearch's `@Field` annotation. Please see <<elasticsearch.mapping.meta-model.annotations>> for detailed information.
+
+[[elasticsearch-migration-guide-3.2-4.0.implicit-index-name]]
+== Removal of implicit index name from query objects
+
+In 3.2.x the different query classes like `IndexQuery` or `SearchQuery` had properties that were taking the index name or index names that they were operating upon. If these were not set, the passed in entity was inspected to retrieve the index name that was set in the `@Document` annotation. +
+In 4.0.x the index name(s) must now be provided in an additional parameter of type `IndexCoordinates`. By separating this, it now is possible to use one query object against different indices.
+
+So for example the following code:
+
+[source,java]
+----
+IndexQuery indexQuery = new IndexQueryBuilder()
+  .withId(person.getId().toString())
+  .withObject(person)
+  .build();
+
+String documentId = elasticsearchOperations.index(indexQuery);
+----
+
+must be changed to:
+
+[source,java]
+----
+IndexCoordinates indexCoordinates = elasticsearchOperations.getIndexCoordinatesFor(person.getClass());
+
+IndexQuery indexQuery = new IndexQueryBuilder()
+  .withId(person.getId().toString())
+  .withObject(person)
+  .build();
+
+String documentId = elasticsearchOperations.index(indexQuery, indexCoordinates);
+----
+
+To make it easier to work with entities and use the index name that is contained in the entitie's `@Document` annotation, new methods have been added like `DocumentOperations.save(T entity)`;
+
+[[elasticsearch-migration-guide-3.2-4.0.new-operations]]
+== The new Operations interfaces
+
+In version 3.2 there was the `ElasticsearchOperations` interface that defined all the methods for the `ElasticsearchTemplate` class. In version 4 the functions have been split into different interfaces, aligning these interfaces with the Elasticsearch API:
+
+* `DocumentOperations` are the functions related documents like saving, or deleting
+* `SearchOperations` contains the functions to search in Elasticsearch
+* `IndexOperations` define the functions to operate on indexes, like index creation or mappings creation.
+
+`ElasticsearchOperations` now extends `DocumentOperations` and `SearchOperations` and has methods get access to an `IndexOperations` instance.
+
+NOTE: All the functions from the `ElasticsearchOperations` interface in version 3.2 that are now moved to the `IndexOperations` interface are still available, they are marked as deprecated and have default implementations that delegate to the new implementation:
+
+[source,java]
+----
+/**
+ * Create an index for given indexName.
+ *
+ * @param indexName the name of the index
+ * @return {@literal true} if the index was created
+ * @deprecated since 4.0, use {@link IndexOperations#create()}
+ */
+@Deprecated
+default boolean createIndex(String indexName) {
+	return indexOps(IndexCoordinates.of(indexName)).create();
+}
+----
+
+[[elasticsearch-migration-guide-3.2-4.0.deprecations]]
+== Deprecations
+
+=== Methods and classes
+
+Many functions and classes have been deprecated. These functions still work, but the Javadocs show with what they should be replaced.
+
+.Example from ElasticsearchOperations
+[source,java]
+----
+/*
+ * Retrieves an object from an index.
+ *
+ * @param query the query defining the id of the object to get
+ * @param clazz the type of the object to be returned
+ * @return the found object
+ * @deprecated since 4.0, use {@link #get(String, Class, IndexCoordinates)}
+ */
+@Deprecated
+@Nullable
+<T> T queryForObject(GetQuery query, Class<T> clazz);
+----
+
+=== Elasticsearch deprecations
+
+Since version 7 the Elasticsearch `TransportClient` is deprecated, it will be removed with Elasticsearch version 8. Spring Data Elasticsearch deprecates the `ElasticsearchTemplate` class which uses the `TransportClient` in version 4.0.
+
+Mapping types were removed from Elasticsearch 7, they still exist as deprecated values in the Spring Data `@Document` annotation and the `IndexCoordinates` class but they are not used anymore internally.
+
+[[elasticsearch-migration-guide-3.2-4.0.removal]]
+== Removals
+
+* As already described, the `ElasticsearchEntityMapper` interface has been removed.
+
+* The `SearchQuery` interface has been merged into it's base interface `Query`, so it's occurrences can just be replaced with `Query`.
+
+* The method `org.springframework.data.elasticsearch.core.ElasticsearchOperations.query(SearchQuery query, ResultsExtractor<T> resultsExtractor);` and the `org.springframework.data.elasticsearch.core.ResultsExtractor` interface have been removed. These could be used to parse the result from Elasticsearch for cases in which the response mapping done with the Jackson based mapper was not enough. Since version 4.0, there are the new <<elasticsearch.operations.searchresulttypes>>  to return the information from an Elasticsearch response, so there is no need to expose this low level functionality.
+
+* The low level methods `startScroll`, `continueScroll` and `clearScroll` have been removed from the `ElasticsearchOperations` interface. For low level scroll API access, there now are `searchScrollStart`, `searchScrollContinue` and `searchScrollClear` methods on the `ElasticsearchRestTemplate` class.

src/main/asciidoc/reference/elasticsearch-misc.adoc

@@ -27,59 +27,65 @@ Page<SampleEntity> sampleEntities = operations.searchForPage(searchQuery, Sample
 [[elasticsearch.scroll]]
 == Using Scroll For Big Result Set
 
-Elasticsearch has a scroll API for getting big result set in chunks. `ElasticsearchOperations` has startScroll and continueScroll methods that can be used as below.
+Elasticsearch has a scroll API for getting big result set in chunks. This is internally used by Spring Data Elasticsearch to provide the implementations of the `<T> SearchHitsIterator<T> SearchOperations.searchForStream(Query query, Class<T> clazz, IndexCoordinates index)` method.
 
-.Using startScroll and continueScroll
-====
 [source,java]
 ----
 IndexCoordinates index = IndexCoordinates.of("sample-index");
 
 SearchQuery searchQuery = new NativeSearchQueryBuilder()
   .withQuery(matchAllQuery())
-  .withIndices(INDEX_NAME)
-  .withTypes(TYPE_NAME)
   .withFields("message")
   .withPageable(PageRequest.of(0, 10))
   .build();
 
-ScrolledPage<SampleEntity> scroll = operations.startScroll(1000, searchQuery, SampleEntity.class, index);
-
-String scrollId = scroll.getScrollId();
-List<SampleEntity> sampleEntities = new ArrayList<>();
-while (scroll.hasContent()) {
-  sampleEntities.addAll(scroll.getContent());
-  scrollId = scroll.getScrollId();
-  scroll = operations.continueScroll(scrollId, 1000, SampleEntity.class);
-}
-operations.clearScroll(scrollId);
-----
-====
-
-`ElasticsearchOperations` additionally has the stream method which wraps the scan and scroll operations into a CloseableIterator.
-
-.Using stream
-====
-[source,java]
-----
-IndexCoordinates index = IndexCoordinates.of("sample-index");
-
-SearchQuery searchQuery = new NativeSearchQueryBuilder()
-  .withQuery(matchAllQuery())
-  .withIndices(INDEX_NAME)
-  .withTypes(TYPE_NAME)
-  .withFields("message")
-  .withPageable(PageRequest.of(0, 10))
-  .build();
-
-CloseableIterator<SampleEntity> stream = elasticsearchTemplate.stream(searchQuery, SampleEntity.class, index);
+SearchHitsIterator<SampleEntity> stream = elasticsearchTemplate.searchForStream(searchQuery, SampleEntity.class, index);
 
 List<SampleEntity> sampleEntities = new ArrayList<>();
 while (stream.hasNext()) {
   sampleEntities.add(stream.next());
 }
+
+stream.close();
+----
+
+There are no methods in the `SearchOperations` API to access the scroll id, if it should be necessary to access this, the following methods of the `ElasticsearchRestTemplate` can be used:
+
+[source,java]
+----
+
+@Autowired ElasticsearchRestTemplate template;
+
+IndexCoordinates index = IndexCoordinates.of("sample-index");
+
+SearchQuery searchQuery = new NativeSearchQueryBuilder()
+  .withQuery(matchAllQuery())
+  .withFields("message")
+  .withPageable(PageRequest.of(0, 10))
+  .build();
+
+SearchScrollHits<SampleEntity> scroll = template.searchScrollStart(1000, searchQuery, SampleEntity.class, index);
+
+String scrollId = scroll.getScrollId();
+List<SampleEntity> sampleEntities = new ArrayList<>();
+while (scroll.hasSearchHits()) {
+  sampleEntities.addAll(scroll.getSearchHits());
+  scrollId = scroll.getScrollId();
+  scroll = template.searchScrollContinue(scrollId, 1000, SampleEntity.class);
+}
+template.searchScrollClear(scrollId);
+----
+
+To use the Scroll API with repository methods, the return type must defined as `Stream` in the Elasticsearch Repository. The implementation of the method will then use the scroll methods from the ElasticsearchTemplate.
+
+[source,java]
+----
+interface SampleEntityRepository extends Repository<SampleEntity, String> {
+
+    Stream<SampleEntity> findBy();
+
+}
 ----
-====
 
 [[elasticsearch.misc.sorts]]
 == Sort options

src/main/asciidoc/reference/elasticsearch-new.adoc

@@ -5,14 +5,18 @@
 == New in Spring Data Elasticsearch 4.0
 
 * Uses Spring 5.2.
-* Upgrade to Elasticsearch 7.6.1.
+* Upgrade to Elasticsearch 7.6.2.
 * Deprecation of `TransportClient` usage.
 * Implements most of the mapping-types available for the index mappings.
 * Removal of the Jackson `ObjectMapper`, now using the <<elasticsearch.mapping.meta-model,MappingElasticsearchConverter>>
 * Cleanup of the API in the `*Operations` interfaces, grouping and renaming methods so that they match the Elasticsearch API, deprecating the old methods, aligning with other Spring Data modules.
 * Introduction of `SearchHit<T>` class to represent a found document together with the relevant result metadata for this document (i.e. _sortValues_).
-* Introduction of the `SearchHits<T>` class to represent a whole search result together with the metadata for the complete search result (i.e. _max_score_).
+* Introduction of the `SearchHits<T>` class to represent a whole search result together with the
+metadata for the complete search result (i.e. _max_score_).
+* Introduction of `SearchPage<T>` class to represent a paged result containing a `SearchHits<T>` instance.
 * Introduction of the `GeoDistanceOrder` class to be able to create sorting by geographical distance
+* Implementation of Auditing Support
+* Implementation of lifecycle entity callbacks
 
 [[new-features.3-2-0]]
 == New in Spring Data Elasticsearch 3.2

src/main/asciidoc/reference/elasticsearch-object-mapping.adoc

@@ -9,7 +9,7 @@ The main reasons for the removal of the Jackson based mapper are:
 
 * Custom mappings of fields needed to be done with annotations like `@JsonFormat` or `@JsonInclude`. This often caused problems when the same object was used in different JSON based datastores or sent over a JSON based API.
 * Custom field types and formats also need to be stored into the Elasticsearch index mappings. The Jackson based annotations did not fully provide all the information that is necessary to represent the types of Elasticsearch.
-* Fields must be mapped not only when converting fromand to entities, but also an query argument, returned data and on other places.
+* Fields must be mapped not only when converting from and to entities, but also in query argument, returned data and on other places.
 
 Using the `MappingElasticsearchConverter` now covers all these cases.
 
@@ -29,25 +29,28 @@ The following annotations are available:
 
 * `@Document`: Applied at the class level to indicate this class is a candidate for mapping to the database. The most important attributes are:
 ** `indexName`: the name of the index to store this entity in
-** `type`: the mapping type. If not set, the lowercased simple name of the class is used.
+** `type`: [line-through]#the mapping type. If not set, the lowercased simple name of the class is used.# (deprecated since version 4.0)
 ** `shards`: the number of shards for the index.
 ** `replicas`: the number of replicas for the index.
 ** `refreshIntervall`: Refresh interval for the index. Used for index creation. Default value is _"1s"_.
 ** `indexStoreType`:  Index storage type for the index. Used for index creation. Default value is _"fs"_.
 ** `createIndex`: Configuration whether to create an index on repository bootstrapping. Default value is _true_.
 ** `versionType`: Configuration of version management. Default value is _EXTERNAL_.
+
 * `@Id`: Applied at the field level to mark the field used for identity purpose.
-* `@Transient`: By default all private fields are mapped to the document, this annotation excludes the field where it is applied from being stored in the database
+* `@Transient`: By default all fields are mapped to the document when it is stored or retrieved, this annotation excludes the field.
 * `@PersistenceConstructor`: Marks a given constructor - even a package protected one - to use when instantiating the object from the database. Constructor arguments are mapped by name to the key values in the retrieved Document.
-* `@Field`: Applied at the field level and defines properties of the field, most of the attributes map to the respective https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html[Elasticsearch Mapping] definitions:
+* `@Field`: Applied at the field level and defines properties of the field, most of the attributes map to the respective https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html[Elasticsearch Mapping] definitions (the following list is not complete, check the annotation Javadoc for a complete reference):
 ** `name`: The name of the field as it will be represented in the Elasticsearch document, if not set, the Java field name is used.
-** `type`: the field type, can be one of _Text, Integer, Long, Date, Float, Double, Boolean, Object, Auto, Nested, Ip, Attachment, Keyword_. See https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-types.html[Elasticsearch Mapping Types]
-** `format` and `pattern` custom definitions for the _Date_ type.
+** `type`: the field type, can be one of _Text, Keyword, Long, Integer, Short, Byte, Double, Float, Half_Float, Scaled_Float, Date, Date_Nanos, Boolean, Binary, Integer_Range, Float_Range, Long_Range, Double_Range, Date_Range, Ip_Range, Object, Nested, Ip, TokenCount, Percolator, Flattened, Search_As_You_Type_. See https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-types.html[Elasticsearch Mapping Types]
+** `format` and `pattern` definitions for the _Date_ type. `format` must be defined for date types.
 ** `store`: Flag wether the original field value should be store in Elasticsearch, default value is _false_.
 ** `analyzer`, `searchAnalyzer`, `normalizer` for specifying custom custom analyzers and normalizer.
-** `copy_to`: the target field to copy multiple document fields to.
 * `@GeoPoint`: marks a field as _geo_point_ datatype. Can be omitted if the field is an instance of the `GeoPoint` class.
 
+NOTE: Properties that derive from `TemporalAccessor` must either have a `@Field` annotation of type `FieldType.Date` or a custom converter must be registerd for this type. +
+If you are using a custom date format, you need to use _uuuu_ for the year instead of _yyyy_. This is due to a https://www.elastic.co/guide/en/elasticsearch/reference/current/migrate-to-java-time.html#java-time-migration-incompatible-date-formats[change in Elasticsearch 7].
+
 The mapping metadata infrastructure is defined in a separate spring-data-commons project that is technology agnostic.
 
 [[elasticsearch.mapping.meta-model.rules]]

src/main/asciidoc/reference/elasticsearch-operations.adoc

@@ -125,7 +125,7 @@ include::reactive-elasticsearch-operations.adoc[leveloffset=+1]
 
 When a document is retrieved with the methods of the  `DocumentOperations` interface, just the found entity will be returned. When searching with the methods of the `SearchOperations` interface, additional information is available for each entity, for example the _score_ or the _sortValues_ of the found entity.
 
-In order to return this information, each entity is wrapped in a `SearchHit` object that contains this entity-specific additional information. These `SearchHit` objects themselves are returned within a `SearchHits` object which additionally contains informations about the whole search ike the _maxScore_ or requested aggregations.
+In order to return this information, each entity is wrapped in a `SearchHit` object that contains this entity-specific additional information. These `SearchHit` objects themselves are returned within a `SearchHits` object which additionally contains informations about the whole search like the _maxScore_ or requested aggregations. The following classes and interfaces are now available:
 
 .SearchHit<T>
 Contains the following information:
@@ -133,7 +133,7 @@ Contains the following information:
 * Id
 * Score
 * Sort Values
-* Highligth fields
+* Highlight fields
 * The retrieved entity of type <T>
 
 .SearchHits<T>
@@ -145,3 +145,12 @@ Contains the following information:
 * A list of `SearchHit<T>` objects
 * Returned aggregations
 
+.SearchPage<T>
+Defines a Spring Data `Page` that contains a `SearchHits<T>` element and can be used for paging access using repository methods.
+
+.SearchScrollHits<T>
+Returned by the low level scroll API functions in `ElasticsearchRestTemplate`, it enriches a `SearchHits<T>` with the Elasticsearch scroll id.
+
+.SearchHitsIterator<T>
+An Iterator returned by the streaming functions of the `SearchOperations` interface.
+

src/main/asciidoc/reference/elasticsearch-repositories.adoc

@@ -3,8 +3,58 @@
 
 This chapter includes details of the Elasticsearch repository implementation.
 
+.The sample `Book` entity
+====
+[source,java]
+----
+@Document(indexName="books")
+class Book {
+    @Id
+    private String id;
+
+    @Field(type = FieldType.text)
+    private String name;
+
+    @Field(type = FieldType.text)
+    private String summary;
+
+    @Field(type = FieldType.Integer)
+    private Integer price;
+
+	// getter/setter ...
+}
+----
+====
+
 include::elasticsearch-repository-queries.adoc[leveloffset=+1]
 
+include::reactive-elasticsearch-repositories.adoc[leveloffset=+1]
+
+[[elasticsearch.repositories.annotations]]
+== Annotations for repository methods
+
+=== @Highlight
+
+The `@Highlight` annotation on a repository method defines for which fields of the returned entity highlighting should be included. To search for some text in a `Book` 's name or summary and have the found data highlighted, the following repository method can be used:
+
+====
+[source,java]
+----
+interface BookRepository extends Repository<Book, String> {
+
+    @Highlight(fields = {
+        @HighlightField(name = "name"),
+        @HighlightField(name = "summary")
+    })
+    List<SearchHit<Book>> findByNameOrSummary(String text, String summary);
+}
+----
+====
+
+It is possible to define multiple fields to be highlighted like above, and both the `@Highlight` and the `@HighlightField` annotation can further be customized with a `@HighlightParameters` annotation. Check the Javadocs for the possible configuration options.
+
+In the search results the highlight data can be retrieved from the `SearchHit` class.
+
 [[elasticsearch.annotation]]
 == Annotation based configuration
 
@@ -40,7 +90,8 @@ class ProductService {
 }
 ----
 
-<1> The `EnableElasticsearchRepositories` annotation activates the Repository support. If no base package is configured, it will use the one of the configuration class it is put on.
+<1> The `EnableElasticsearchRepositories` annotation activates the Repository support.
+If no base package is configured, it will use the one of the configuration class it is put on.
 <2> Provide a Bean named `elasticsearchTemplate` of type `ElasticsearchOperations` by using one of the configurations shown in the <<elasticsearch.operations>> chapter.
 <3> Let Spring inject the Repository bean into your class.
 ====
@@ -145,5 +196,3 @@ Using the `Transport Client` or `Rest Client` element registers an instance of `
 </beans>
 ----
 ====
-
-include::reactive-elasticsearch-repositories.adoc[leveloffset=+1]

src/main/asciidoc/reference/elasticsearch-repository-queries.adoc

@@ -48,7 +48,9 @@ A list of supported keywords for Elasticsearch is shown below.
 |===
 | Keyword
 | Sample
-| Elasticsearch Query String| `And`
+| Elasticsearch Query String
+
+| `And`
 | `findByNameAndPrice`
 | `{ "query" : {
 "bool" : {
@@ -201,7 +203,7 @@ A list of supported keywords for Elasticsearch is shown below.
   }
 }}`
 
-| `In`
+| `In` (when annotated as FieldType.Keyword)
 | `findByNameIn(Collection<String>names)`
 | `{ "query" : {
 "bool" : {
@@ -215,7 +217,12 @@ A list of supported keywords for Elasticsearch is shown below.
   }
 }}`
 
-| `NotIn`
+
+| `In`
+| `findByNameIn(Collection<String>names)`
+| `{ "query": {"bool": {"must": [{"query_string":{"query": "\"?\" \"?\"", "fields": ["name"]}}]}}}`
+
+| `NotIn`  (when annotated as FieldType.Keyword)
 | `findByNameNotIn(Collection<String>names)`
 | `{ "query" : {
 "bool" : {
@@ -229,6 +236,10 @@ A list of supported keywords for Elasticsearch is shown below.
   }
 }}`
 
+| `NotIn`
+| `findByNameNotIn(Collection<String>names)`
+| `{"query": {"bool": {"must": [{"query_string": {"query": "NOT(\"?\" \"?\")", "fields": ["name"]}}]}}}`
+
 | `Near`
 | `findByStoreNear`
 | `Not Supported Yet !`
@@ -273,7 +284,7 @@ Repository methods can be defined to have the following return types for returni
 * `List<T>`
 * `Stream<T>`
 * `SearchHits<T>`
-* `List<SearchHits<T>>`
+* `List<SearchHit<T>>`
 * `Stream<SearchHit<T>>`
 * `SearchPage<T>`
 

src/main/asciidoc/reference/migration-guides.adoc

@@ -0,0 +1,6 @@
+[[elasticsearch.migration]]
+= Appendix E: Migration Guides
+
+:leveloffset: +1
+include::elasticsearch-migration-guide-3.2-4.0.adoc[]
+:leveloffset: -1

src/main/java/org/springframework/data/elasticsearch/BulkFailureException.java

@@ -0,0 +1,37 @@
+/*
+ * Copyright 2020 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.springframework.data.elasticsearch;
+
+import org.springframework.dao.DataRetrievalFailureException;
+
+import java.util.Map;
+
+/**
+ * @author Peter-Josef Meisch
+ * @since 4.0.1 (ported back from master (4.1) branch)
+ */
+public class BulkFailureException extends DataRetrievalFailureException {
+	private final Map<String, String> failedDocuments;
+
+	public BulkFailureException(String msg, Map<String, String> failedDocuments) {
+		super(msg);
+		this.failedDocuments = failedDocuments;
+	}
+
+	public Map<String, String> getFailedDocuments() {
+		return failedDocuments;
+	}
+}

src/main/java/org/springframework/data/elasticsearch/ElasticsearchException.java

@@ -25,7 +25,9 @@ import org.springframework.lang.Nullable;
  * @author Rizwan Idrees
  * @author Mohsin Husen
  * @author Peter-Josef Meisch
+ * @deprecated since 4.0, use {@link org.springframework.dao.UncategorizedDataAccessException}
  */
+@Deprecated
 public class ElasticsearchException extends RuntimeException {
 
 	@Nullable private Map<String, String> failedDocuments;

src/main/java/org/springframework/data/elasticsearch/UncategorizedElasticsearchException.java

@@ -13,27 +13,17 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.springframework.data.elasticsearch.support;
+package org.springframework.data.elasticsearch;
 
-import java.util.concurrent.atomic.AtomicBoolean;
-
-import org.springframework.data.repository.util.ClassUtils;
+import org.springframework.dao.UncategorizedDataAccessException;
 
 /**
  * @author Peter-Josef Meisch
  * @since 4.0
  */
-public final class ReactiveSupport {
-	private ReactiveSupport() {}
+public class UncategorizedElasticsearchException extends UncategorizedDataAccessException {
 
-	/**
-	 * @return true if project reactor is on the classpath
-	 */
-	public static boolean isReactorAvailable() {
-		AtomicBoolean available = new AtomicBoolean(false);
-		ClassUtils.ifPresent("reactor.core.publisher.Flux", null, aClass -> {
-			available.set(true);
-		});
-		return available.get();
+	public UncategorizedElasticsearchException(String msg, Throwable cause) {
+		super(msg, cause);
 	}
 }

src/main/java/org/springframework/data/elasticsearch/annotations/CompletionContext.java

@@ -1,7 +1,5 @@
 package org.springframework.data.elasticsearch.annotations;
 
-import org.elasticsearch.search.suggest.completion.context.ContextMapping;
-
 import java.lang.annotation.Documented;
 import java.lang.annotation.ElementType;
 import java.lang.annotation.Inherited;
@@ -9,6 +7,8 @@ import java.lang.annotation.Retention;
 import java.lang.annotation.RetentionPolicy;
 import java.lang.annotation.Target;
 
+import org.elasticsearch.search.suggest.completion.context.ContextMapping;
+
 /**
  * Based on reference doc - https://www.elastic.co/guide/en/elasticsearch/reference/current/suggester-context.html
  *
@@ -26,4 +26,5 @@ public @interface CompletionContext {
 
 	String precision() default "";
 
+	String path() default "";
 }

src/main/java/org/springframework/data/elasticsearch/annotations/Field.java

@@ -33,6 +33,7 @@ import org.springframework.core.annotation.AliasFor;
  * @author Kevin Leturc
  * @author Peter-Josef Meisch
  * @author Xiao Yu
+ * @author Aleksei Arsenev
  */
 @Retention(RetentionPolicy.RUNTIME)
 @Target(ElementType.FIELD)
@@ -148,4 +149,9 @@ public @interface Field {
 	 * @since 4.0
 	 */
 	double scalingFactor() default 1;
+
+	/**
+	 * @since 4.0
+	 */
+	int maxShingleSize() default -1;
 }

src/main/java/org/springframework/data/elasticsearch/annotations/FieldType.java

@@ -21,33 +21,35 @@ package org.springframework.data.elasticsearch.annotations;
  * @author Artur Konczak
  * @author Zeng Zetang
  * @author Peter-Josef Meisch
+ * @author Aleksei Arsenev
  */
 public enum FieldType {
-    Auto,
-    Text,
-    Keyword,
-    Long,
-    Integer,
-    Short,
-    Byte,
-    Double,
-    Float,
-    Half_Float,
-    Scaled_Float,
-    Date,
-    Date_Nanos,
-    Boolean,
-    Binary,
-    Integer_Range,
-    Float_Range,
-    Long_Range,
-    Double_Range,
-    Date_Range,
-    Ip_Range,
-    Object,
-    Nested,
-    Ip,
-    TokenCount,
-    Percolator,
-    Flattened
+    Auto, //
+    Text, //
+    Keyword, //
+    Long, //
+    Integer, //
+    Short, //
+    Byte, //
+    Double, //
+    Float, //
+    Half_Float, //
+    Scaled_Float, //
+    Date, //
+    Date_Nanos, //
+    Boolean, //
+    Binary, //
+    Integer_Range, //
+    Float_Range, //
+    Long_Range, //
+    Double_Range, //
+    Date_Range, //
+    Ip_Range, //
+    Object, //
+    Nested, //
+    Ip, //
+    TokenCount, //
+    Percolator, //
+    Flattened, //
+    Search_As_You_Type //
 }

src/main/java/org/springframework/data/elasticsearch/annotations/InnerField.java

@@ -26,9 +26,10 @@ import java.lang.annotation.Target;
  * @author Sascha Woo
  * @author Xiao Yu
  * @author Peter-Josef Meisch
+ * @author Aleksei Arsenev
  */
 @Retention(RetentionPolicy.RUNTIME)
-@Target(ElementType.FIELD)
+@Target(ElementType.ANNOTATION_TYPE)
 public @interface InnerField {
 
 	String suffix();
@@ -117,4 +118,9 @@ public @interface InnerField {
 	 * @since 4.0
 	 */
 	double scalingFactor() default 1;
+
+	/**
+	 * @since 4.0
+	 */
+	int maxShingleSize() default -1;
 }

src/main/java/org/springframework/data/elasticsearch/annotations/TermVector.java

@@ -20,5 +20,5 @@ package org.springframework.data.elasticsearch.annotations;
  * @since 4.0
  */
 public enum TermVector {
-	none, no, yes, with_positions, with_offsets, woth_positions_offsets, with_positions_payloads, with_positions_offets_payloads
+	none, no, yes, with_positions, with_offsets, with_positions_offsets, with_positions_payloads, with_positions_offsets_payloads
 }

src/main/java/org/springframework/data/elasticsearch/client/ClientConfiguration.java

@@ -21,11 +21,13 @@ import java.time.Duration;
 import java.util.List;
 import java.util.Optional;
 import java.util.function.Function;
+import java.util.function.Supplier;
 
 import javax.net.ssl.HostnameVerifier;
 import javax.net.ssl.SSLContext;
 
 import org.springframework.http.HttpHeaders;
+import org.springframework.lang.Nullable;
 import org.springframework.web.reactive.function.client.WebClient;
 
 /**
@@ -153,6 +155,7 @@ public interface ClientConfiguration {
 	 * @return the path prefix.
 	 * @since 4.0
 	 */
+	@Nullable
 	String getPathPrefix();
 
 	/**
@@ -168,6 +171,11 @@ public interface ClientConfiguration {
 	 */
 	Function<WebClient, WebClient> getWebClientConfigurer();
 
+	/**
+	 * @return the supplier for custom headers.
+	 */
+	Supplier<HttpHeaders> getHeadersSupplier();
+
 	/**
 	 * @author Christoph Strobl
 	 */
@@ -333,6 +341,19 @@ public interface ClientConfiguration {
 		 */
 		TerminalClientConfigurationBuilder withWebClientConfigurer(Function<WebClient, WebClient> webClientConfigurer);
 
+		/**
+		 * set a supplier for custom headers. This is invoked for every HTTP request to Elasticsearch to retrieve headers
+		 * that should be sent with the request. A common use case is passing in authentication headers that may change.
+		 * <br/>
+		 * Note: When used in a reactive environment, the calling of {@link Supplier#get()} function must not do any
+		 * blocking operations. It may return {@literal null}.
+		 * 
+		 * @param headers supplier function for headers, must not be {@literal null}
+		 * @return the {@link TerminalClientConfigurationBuilder}.
+		 * @since 4.0
+		 */
+		TerminalClientConfigurationBuilder withHeaders(Supplier<HttpHeaders> headers);
+
 		/**
 		 * Build the {@link ClientConfiguration} object.
 		 *

src/main/java/org/springframework/data/elasticsearch/client/ClientConfigurationBuilder.java

@@ -21,6 +21,7 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 import java.util.function.Function;
+import java.util.function.Supplier;
 import java.util.stream.Collectors;
 
 import javax.net.ssl.HostnameVerifier;
@@ -58,7 +59,8 @@ class ClientConfigurationBuilder
 	private @Nullable String password;
 	private @Nullable String pathPrefix;
 	private @Nullable String proxy;
-	private @Nullable Function<WebClient, WebClient> webClientConfigurer;
+	private Function<WebClient, WebClient> webClientConfigurer = Function.identity();
+	private Supplier<HttpHeaders> headersSupplier = () -> HttpHeaders.EMPTY;
 
 	/*
 	 * (non-Javadoc)
@@ -196,7 +198,8 @@ class ClientConfigurationBuilder
 	}
 
 	@Override
-	public TerminalClientConfigurationBuilder withWebClientConfigurer(Function<WebClient, WebClient> webClientConfigurer) {
+	public TerminalClientConfigurationBuilder withWebClientConfigurer(
+			Function<WebClient, WebClient> webClientConfigurer) {
 
 		Assert.notNull(webClientConfigurer, "webClientConfigurer must not be null");
 
@@ -204,6 +207,15 @@ class ClientConfigurationBuilder
 		return this;
 	}
 
+	@Override
+	public TerminalClientConfigurationBuilder withHeaders(Supplier<HttpHeaders> headers) {
+
+		Assert.notNull(headers, "headersSupplier must not be null");
+
+		this.headersSupplier = headers;
+		return this;
+	}
+
 	/*
 	 * (non-Javadoc)
 	 * @see org.springframework.data.elasticsearch.client.ClientConfiguration.ClientConfigurationBuilderWithOptionalDefaultHeaders#build()
@@ -219,7 +231,7 @@ class ClientConfigurationBuilder
 		}
 
 		return new DefaultClientConfiguration(hosts, headers, useSsl, sslContext, soTimeout, connectTimeout, pathPrefix,
-				hostnameVerifier, proxy, webClientConfigurer);
+				hostnameVerifier, proxy, webClientConfigurer, headersSupplier);
 	}
 
 	private static InetSocketAddress parse(String hostAndPort) {

src/main/java/org/springframework/data/elasticsearch/client/ClientLogger.java

@@ -20,6 +20,7 @@ import java.util.function.Supplier;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.http.HttpStatus;
+import org.springframework.lang.Nullable;
 import org.springframework.util.ObjectUtils;
 
 /**
@@ -90,7 +91,7 @@ public abstract class ClientLogger {
 	 * @param logId the correlation Id, see {@link #newLogId()}.
 	 * @param statusCode the HTTP status code.
 	 */
-	public static void logRawResponse(String logId, HttpStatus statusCode) {
+	public static void logRawResponse(String logId, @Nullable HttpStatus statusCode) {
 
 		if (isEnabled()) {
 			WIRE_LOGGER.trace("[{}] Received raw response: {}", logId, statusCode);

src/main/java/org/springframework/data/elasticsearch/client/DefaultClientConfiguration.java

@@ -22,6 +22,7 @@ import java.util.Collections;
 import java.util.List;
 import java.util.Optional;
 import java.util.function.Function;
+import java.util.function.Supplier;
 
 import javax.net.ssl.HostnameVerifier;
 import javax.net.ssl.SSLContext;
@@ -47,14 +48,16 @@ class DefaultClientConfiguration implements ClientConfiguration {
 	private final @Nullable SSLContext sslContext;
 	private final Duration soTimeout;
 	private final Duration connectTimeout;
-	private final String pathPrefix;
+	private final @Nullable String pathPrefix;
 	private final @Nullable HostnameVerifier hostnameVerifier;
-	private final String proxy;
+	private final @Nullable String proxy;
 	private final Function<WebClient, WebClient> webClientConfigurer;
+	private final Supplier<HttpHeaders> headersSupplier;
 
 	DefaultClientConfiguration(List<InetSocketAddress> hosts, HttpHeaders headers, boolean useSsl,
 			@Nullable SSLContext sslContext, Duration soTimeout, Duration connectTimeout, @Nullable String pathPrefix,
-			@Nullable HostnameVerifier hostnameVerifier, String proxy, Function<WebClient, WebClient> webClientConfigurer) {
+			@Nullable HostnameVerifier hostnameVerifier, @Nullable String proxy,
+			Function<WebClient, WebClient> webClientConfigurer, Supplier<HttpHeaders> headersSupplier) {
 
 		this.hosts = Collections.unmodifiableList(new ArrayList<>(hosts));
 		this.headers = new HttpHeaders(headers);
@@ -66,6 +69,7 @@ class DefaultClientConfiguration implements ClientConfiguration {
 		this.hostnameVerifier = hostnameVerifier;
 		this.proxy = proxy;
 		this.webClientConfigurer = webClientConfigurer;
+		this.headersSupplier = headersSupplier;
 	}
 
 	@Override
@@ -103,6 +107,7 @@ class DefaultClientConfiguration implements ClientConfiguration {
 		return this.soTimeout;
 	}
 
+	@Nullable
 	@Override
 	public String getPathPrefix() {
 		return this.pathPrefix;
@@ -115,6 +120,11 @@ class DefaultClientConfiguration implements ClientConfiguration {
 
 	@Override
 	public Function<WebClient, WebClient> getWebClientConfigurer() {
-		return webClientConfigurer != null ? webClientConfigurer : Function.identity();
+		return webClientConfigurer;
+	}
+
+	@Override
+	public Supplier<HttpHeaders> getHeadersSupplier() {
+		return headersSupplier;
 	}
 }

src/main/java/org/springframework/data/elasticsearch/client/RestClientFactoryBean.java

@@ -15,14 +15,14 @@
  */
 package org.springframework.data.elasticsearch.client;
 
-import lombok.extern.slf4j.Slf4j;
-
 import java.net.URL;
 import java.util.ArrayList;
 
 import org.apache.http.HttpHost;
 import org.elasticsearch.client.RestClient;
 import org.elasticsearch.client.RestHighLevelClient;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.DisposableBean;
 import org.springframework.beans.factory.FactoryBean;
 import org.springframework.beans.factory.FactoryBeanNotInitializedException;
@@ -36,9 +36,10 @@ import org.springframework.util.Assert;
  * @author Don Wellington
  * @author Peter-Josef Meisch
  */
-@Slf4j
 public class RestClientFactoryBean implements FactoryBean<RestHighLevelClient>, InitializingBean, DisposableBean {
 
+	private static final Logger LOGGER = LoggerFactory.getLogger(RestClientFactoryBean.class);
+
 	private @Nullable RestHighLevelClient client;
 	private String hosts = "http://localhost:9200";
 	static final String COMMA = ",";
@@ -46,12 +47,12 @@ public class RestClientFactoryBean implements FactoryBean<RestHighLevelClient>,
 	@Override
 	public void destroy() {
 		try {
-			log.info("Closing elasticSearch  client");
+			LOGGER.info("Closing elasticSearch  client");
 			if (client != null) {
 				client.close();
 			}
 		} catch (final Exception e) {
-			log.error("Error closing ElasticSearch client: ", e);
+			LOGGER.error("Error closing ElasticSearch client: ", e);
 		}
 	}
 

src/main/java/org/springframework/data/elasticsearch/client/RestClients.java

@@ -20,13 +20,11 @@ import java.io.Closeable;
 import java.io.IOException;
 import java.net.InetSocketAddress;
 import java.time.Duration;
+import java.util.Arrays;
 import java.util.List;
-import java.util.Optional;
+import java.util.function.Supplier;
 import java.util.stream.Collectors;
 
-import javax.net.ssl.HostnameVerifier;
-import javax.net.ssl.SSLContext;
-
 import org.apache.http.Header;
 import org.apache.http.HttpEntity;
 import org.apache.http.HttpEntityEnclosingRequest;
@@ -55,6 +53,7 @@ import org.springframework.util.Assert;
  * @author Mark Paluch
  * @author Huw Ayling-Miller
  * @author Henrique Amaral
+ * @author Peter-Josef Meisch
  * @since 3.2
  */
 public final class RestClients {
@@ -86,31 +85,23 @@ public final class RestClients {
 		HttpHeaders headers = clientConfiguration.getDefaultHeaders();
 
 		if (!headers.isEmpty()) {
-
-			Header[] httpHeaders = headers.toSingleValueMap().entrySet().stream()
-					.map(it -> new BasicHeader(it.getKey(), it.getValue())).toArray(Header[]::new);
-			builder.setDefaultHeaders(httpHeaders);
+			builder.setDefaultHeaders(toHeaderArray(headers));
 		}
 
 		builder.setHttpClientConfigCallback(clientBuilder -> {
-
-			Optional<SSLContext> sslContext = clientConfiguration.getSslContext();
-			Optional<HostnameVerifier> hostNameVerifier = clientConfiguration.getHostNameVerifier();
-			sslContext.ifPresent(clientBuilder::setSSLContext);
-			hostNameVerifier.ifPresent(clientBuilder::setSSLHostnameVerifier);
+			clientConfiguration.getSslContext().ifPresent(clientBuilder::setSSLContext);
+			clientConfiguration.getHostNameVerifier().ifPresent(clientBuilder::setSSLHostnameVerifier);
+			clientBuilder.addInterceptorLast(new CustomHeaderInjector(clientConfiguration.getHeadersSupplier()));
 
 			if (ClientLogger.isEnabled()) {
-
 				HttpLoggingInterceptor interceptor = new HttpLoggingInterceptor();
 
 				clientBuilder.addInterceptorLast((HttpRequestInterceptor) interceptor);
 				clientBuilder.addInterceptorLast((HttpResponseInterceptor) interceptor);
 			}
 
-			Duration connectTimeout = clientConfiguration.getConnectTimeout();
-			Duration timeout = clientConfiguration.getSocketTimeout();
-
 			Builder requestConfigBuilder = RequestConfig.custom();
+			Duration connectTimeout = clientConfiguration.getConnectTimeout();
 
 			if (!connectTimeout.isNegative()) {
 
@@ -118,6 +109,8 @@ public final class RestClients {
 				requestConfigBuilder.setConnectionRequestTimeout(Math.toIntExact(connectTimeout.toMillis()));
 			}
 
+			Duration timeout = clientConfiguration.getSocketTimeout();
+
 			if (!timeout.isNegative()) {
 				requestConfigBuilder.setSocketTimeout(Math.toIntExact(timeout.toMillis()));
 			}
@@ -133,8 +126,16 @@ public final class RestClients {
 		return () -> client;
 	}
 
+	private static Header[] toHeaderArray(HttpHeaders headers) {
+		return headers.entrySet().stream() //
+				.flatMap(entry -> entry.getValue().stream() //
+						.map(value -> new BasicHeader(entry.getKey(), value))) //
+				.toArray(Header[]::new);
+	}
+
 	private static List<String> formattedHosts(List<InetSocketAddress> hosts, boolean useSsl) {
-		return hosts.stream().map(it -> (useSsl ? "https" : "http") + "://" + it).collect(Collectors.toList());
+		return hosts.stream().map(it -> (useSsl ? "https" : "http") + "://" + it.getHostString() + ":" + it.getPort())
+				.collect(Collectors.toList());
 	}
 
 	/**
@@ -179,7 +180,6 @@ public final class RestClients {
 			String logId = (String) context.getAttribute(RestClients.LOG_ID_ATTRIBUTE);
 
 			if (logId == null) {
-
 				logId = ClientLogger.newLogId();
 				context.setAttribute(RestClients.LOG_ID_ATTRIBUTE, logId);
 			}
@@ -204,10 +204,31 @@ public final class RestClients {
 
 		@Override
 		public void process(HttpResponse response, HttpContext context) {
-
 			String logId = (String) context.getAttribute(RestClients.LOG_ID_ATTRIBUTE);
-
 			ClientLogger.logRawResponse(logId, HttpStatus.resolve(response.getStatusLine().getStatusCode()));
 		}
 	}
+
+	/**
+	 * Interceptor to inject custom supplied headers.
+	 * 
+	 * @since 4.0
+	 */
+	private static class CustomHeaderInjector implements HttpRequestInterceptor {
+
+		public CustomHeaderInjector(Supplier<HttpHeaders> headersSupplier) {
+			this.headersSupplier = headersSupplier;
+		}
+
+		private final Supplier<HttpHeaders> headersSupplier;
+
+		@Override
+		public void process(HttpRequest request, HttpContext context) {
+			HttpHeaders httpHeaders = headersSupplier.get();
+
+			if (httpHeaders != null && httpHeaders != HttpHeaders.EMPTY) {
+				Arrays.stream(toHeaderArray(httpHeaders)).forEach(request::addHeader);
+			}
+		}
+	}
 }

src/main/java/org/springframework/data/elasticsearch/client/reactive/DefaultReactiveElasticsearchClient.java

@@ -16,7 +16,9 @@
 package org.springframework.data.elasticsearch.client.reactive;
 
 import io.netty.channel.ChannelOption;
+import io.netty.handler.ssl.ApplicationProtocolConfig;
 import io.netty.handler.ssl.ClientAuth;
+import io.netty.handler.ssl.IdentityCipherSuiteFilter;
 import io.netty.handler.ssl.JdkSslContext;
 import io.netty.handler.timeout.ReadTimeoutHandler;
 import io.netty.handler.timeout.WriteTimeoutHandler;
@@ -34,18 +36,17 @@ import java.net.ConnectException;
 import java.net.InetSocketAddress;
 import java.nio.charset.StandardCharsets;
 import java.time.Duration;
-import java.util.ArrayList;
 import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
 import java.util.Map.Entry;
 import java.util.Optional;
 import java.util.concurrent.TimeUnit;
 import java.util.function.Function;
+import java.util.function.Supplier;
 
 import javax.net.ssl.SSLContext;
 
 import org.apache.http.util.EntityUtils;
+import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ElasticsearchStatusException;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.admin.indices.close.CloseIndexRequest;
@@ -79,7 +80,6 @@ import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.action.update.UpdateRequest;
 import org.elasticsearch.action.update.UpdateResponse;
 import org.elasticsearch.client.Request;
-import org.elasticsearch.client.core.CountRequest;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.DeprecationHandler;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
@@ -90,18 +90,18 @@ import org.elasticsearch.index.reindex.BulkByScrollResponse;
 import org.elasticsearch.index.reindex.DeleteByQueryRequest;
 import org.elasticsearch.rest.BytesRestResponse;
 import org.elasticsearch.rest.RestStatus;
-import org.elasticsearch.search.Scroll;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.SearchHits;
+import org.elasticsearch.search.aggregations.Aggregation;
 import org.reactivestreams.Publisher;
-import org.springframework.data.elasticsearch.ElasticsearchException;
 import org.springframework.data.elasticsearch.client.ClientConfiguration;
 import org.springframework.data.elasticsearch.client.ClientLogger;
 import org.springframework.data.elasticsearch.client.ElasticsearchHost;
 import org.springframework.data.elasticsearch.client.NoReachableHostException;
 import org.springframework.data.elasticsearch.client.reactive.HostProvider.Verification;
 import org.springframework.data.elasticsearch.client.reactive.ReactiveElasticsearchClient.Indices;
-import org.springframework.data.elasticsearch.client.util.RequestConverters;
+import org.springframework.data.elasticsearch.client.util.NamedXContents;
+import org.springframework.data.elasticsearch.client.util.ScrollState;
 import org.springframework.data.util.Lazy;
 import org.springframework.http.HttpHeaders;
 import org.springframework.http.HttpMethod;
@@ -112,8 +112,6 @@ import org.springframework.lang.Nullable;
 import org.springframework.util.Assert;
 import org.springframework.util.ObjectUtils;
 import org.springframework.util.ReflectionUtils;
-import org.springframework.util.StringUtils;
-import org.springframework.web.client.HttpServerErrorException;
 import org.springframework.web.reactive.function.BodyExtractors;
 import org.springframework.web.reactive.function.client.ClientRequest;
 import org.springframework.web.reactive.function.client.ClientResponse;
@@ -128,6 +126,8 @@ import org.springframework.web.reactive.function.client.WebClient.RequestBodySpe
  * @author Peter-Josef Meisch
  * @author Huw Ayling-Miller
  * @author Henrique Amaral
+ * @author Roman Puchkovskiy
+ * @author Russell Parry
  * @since 3.2
  * @see ClientConfiguration
  * @see ReactiveRestClients
@@ -135,6 +135,8 @@ import org.springframework.web.reactive.function.client.WebClient.RequestBodySpe
 public class DefaultReactiveElasticsearchClient implements ReactiveElasticsearchClient, Indices {
 
 	private final HostProvider hostProvider;
+	private final RequestCreator requestCreator;
+	private Supplier<HttpHeaders> headersSupplier = () -> HttpHeaders.EMPTY;
 
 	/**
 	 * Create a new {@link DefaultReactiveElasticsearchClient} using the given {@link HostProvider} to obtain server
@@ -143,10 +145,30 @@ public class DefaultReactiveElasticsearchClient implements ReactiveElasticsearch
 	 * @param hostProvider must not be {@literal null}.
 	 */
 	public DefaultReactiveElasticsearchClient(HostProvider hostProvider) {
+		this(hostProvider, new DefaultRequestCreator());
+	}
+
+	/**
+	 * Create a new {@link DefaultReactiveElasticsearchClient} using the given {@link HostProvider} to obtain server
+	 * connections and the given {@link RequestCreator}.
+	 *
+	 * @param hostProvider must not be {@literal null}.
+	 * @param requestCreator must not be {@literal null}.
+	 */
+	public DefaultReactiveElasticsearchClient(HostProvider hostProvider, RequestCreator requestCreator) {
 
 		Assert.notNull(hostProvider, "HostProvider must not be null");
+		Assert.notNull(requestCreator, "RequestCreator must not be null");
 
 		this.hostProvider = hostProvider;
+		this.requestCreator = requestCreator;
+	}
+
+	public void setHeadersSupplier(Supplier<HttpHeaders> headersSupplier) {
+
+		Assert.notNull(headersSupplier, "headersSupplier must not be null");
+
+		this.headersSupplier = headersSupplier;
 	}
 
 	/**
@@ -177,14 +199,35 @@ public class DefaultReactiveElasticsearchClient implements ReactiveElasticsearch
 	 * @return new instance of {@link DefaultReactiveElasticsearchClient}.
 	 */
 	public static ReactiveElasticsearchClient create(ClientConfiguration clientConfiguration) {
+		return create(clientConfiguration, new DefaultRequestCreator());
+	}
+
+	/**
+	 * Create a new {@link DefaultReactiveElasticsearchClient} given {@link ClientConfiguration} and
+	 * {@link RequestCreator}. <br />
+	 * <strong>NOTE</strong> If the cluster requires authentication be sure to provide the according {@link HttpHeaders}
+	 * correctly.
+	 *
+	 * @param clientConfiguration Client configuration. Must not be {@literal null}.
+	 * @param requestCreator Request creator. Must not be {@literal null}.
+	 * @return new instance of {@link DefaultReactiveElasticsearchClient}.
+	 */
+	public static ReactiveElasticsearchClient create(ClientConfiguration clientConfiguration,
+			RequestCreator requestCreator) {
 
 		Assert.notNull(clientConfiguration, "ClientConfiguration must not be null");
+		Assert.notNull(requestCreator, "RequestCreator must not be null");
 
 		WebClientProvider provider = getWebClientProvider(clientConfiguration);
 
-		HostProvider hostProvider = HostProvider.provider(provider,
+		HostProvider hostProvider = HostProvider.provider(provider, clientConfiguration.getHeadersSupplier(),
 				clientConfiguration.getEndpoints().toArray(new InetSocketAddress[0]));
-		return new DefaultReactiveElasticsearchClient(hostProvider);
+
+		DefaultReactiveElasticsearchClient client = new DefaultReactiveElasticsearchClient(hostProvider, requestCreator);
+
+		client.setHeadersSupplier(clientConfiguration.getHeadersSupplier());
+
+		return client;
 	}
 
 	private static WebClientProvider getWebClientProvider(ClientConfiguration clientConfiguration) {
@@ -220,11 +263,16 @@ public class DefaultReactiveElasticsearchClient implements ReactiveElasticsearch
 
 		if (clientConfiguration.useSsl()) {
 
-			httpClient = httpClient.secure(sslConfig -> {
+			Optional<SSLContext> sslContext = clientConfiguration.getSslContext();
 
-				Optional<SSLContext> sslContext = clientConfiguration.getSslContext();
-				sslContext.ifPresent(it -> sslConfig.sslContext(new JdkSslContext(it, true, ClientAuth.NONE)));
-			});
+			if (sslContext.isPresent()) {
+				httpClient = httpClient.secure(sslContextSpec -> {
+					sslContextSpec.sslContext(new JdkSslContext(sslContext.get(), true, null, IdentityCipherSuiteFilter.INSTANCE,
+							ApplicationProtocolConfig.DISABLED, ClientAuth.NONE, null, false));
+				});
+			} else {
+				httpClient = httpClient.secure();
+			}
 
 			scheme = "https";
 		}
@@ -248,7 +296,7 @@ public class DefaultReactiveElasticsearchClient implements ReactiveElasticsearch
 	@Override
 	public Mono<Boolean> ping(HttpHeaders headers) {
 
-		return sendRequest(new MainRequest(), RequestCreator.ping(), RawActionResponse.class, headers) //
+		return sendRequest(new MainRequest(), requestCreator.ping(), RawActionResponse.class, headers) //
 				.map(response -> response.statusCode().is2xxSuccessful()) //
 				.onErrorResume(NoReachableHostException.class, error -> Mono.just(false)).next();
 	}
@@ -260,7 +308,7 @@ public class DefaultReactiveElasticsearchClient implements ReactiveElasticsearch
 	@Override
 	public Mono<MainResponse> info(HttpHeaders headers) {
 
-		return sendRequest(new MainRequest(), RequestCreator.info(), MainResponse.class, headers) //
+		return sendRequest(new MainRequest(), requestCreator.info(), MainResponse.class, headers) //
 				.next();
 	}
 
@@ -271,7 +319,7 @@ public class DefaultReactiveElasticsearchClient implements ReactiveElasticsearch
 	@Override
 	public Mono<GetResult> get(HttpHeaders headers, GetRequest getRequest) {
 
-		return sendRequest(getRequest, RequestCreator.get(), GetResponse.class, headers) //
+		return sendRequest(getRequest, requestCreator.get(), GetResponse.class, headers) //
 				.filter(GetResponse::isExists) //
 				.map(DefaultReactiveElasticsearchClient::getResponseToGetResult) //
 				.next();
@@ -284,7 +332,7 @@ public class DefaultReactiveElasticsearchClient implements ReactiveElasticsearch
 	@Override
 	public Flux<GetResult> multiGet(HttpHeaders headers, MultiGetRequest multiGetRequest) {
 
-		return sendRequest(multiGetRequest, RequestCreator.multiGet(), MultiGetResponse.class, headers)
+		return sendRequest(multiGetRequest, requestCreator.multiGet(), MultiGetResponse.class, headers)
 				.map(MultiGetResponse::getResponses) //
 				.flatMap(Flux::fromArray) //
 				.filter(it -> !it.isFailed() && it.getResponse().isExists()) //
@@ -298,7 +346,7 @@ public class DefaultReactiveElasticsearchClient implements ReactiveElasticsearch
 	@Override
 	public Mono<Boolean> exists(HttpHeaders headers, GetRequest getRequest) {
 
-		return sendRequest(getRequest, RequestCreator.exists(), RawActionResponse.class, headers) //
+		return sendRequest(getRequest, requestCreator.exists(), RawActionResponse.class, headers) //
 				.map(response -> response.statusCode().is2xxSuccessful()) //
 				.next();
 	}
@@ -309,7 +357,7 @@ public class DefaultReactiveElasticsearchClient implements ReactiveElasticsearch
 	 */
 	@Override
 	public Mono<IndexResponse> index(HttpHeaders headers, IndexRequest indexRequest) {
-		return sendRequest(indexRequest, RequestCreator.index(), IndexResponse.class, headers).publishNext();
+		return sendRequest(indexRequest, requestCreator.index(), IndexResponse.class, headers).publishNext();
 	}
 
 	/*
@@ -327,7 +375,7 @@ public class DefaultReactiveElasticsearchClient implements ReactiveElasticsearch
 	 */
 	@Override
 	public Mono<UpdateResponse> update(HttpHeaders headers, UpdateRequest updateRequest) {
-		return sendRequest(updateRequest, RequestCreator.update(), UpdateResponse.class, headers).publishNext();
+		return sendRequest(updateRequest, requestCreator.update(), UpdateResponse.class, headers).publishNext();
 	}
 
 	/*
@@ -337,7 +385,7 @@ public class DefaultReactiveElasticsearchClient implements ReactiveElasticsearch
 	@Override
 	public Mono<DeleteResponse> delete(HttpHeaders headers, DeleteRequest deleteRequest) {
 
-		return sendRequest(deleteRequest, RequestCreator.delete(), DeleteResponse.class, headers) //
+		return sendRequest(deleteRequest, requestCreator.delete(), DeleteResponse.class, headers) //
 				.publishNext();
 	}
 
@@ -347,7 +395,10 @@ public class DefaultReactiveElasticsearchClient implements ReactiveElasticsearch
 	 */
 	@Override
 	public Mono<Long> count(HttpHeaders headers, SearchRequest searchRequest) {
-		return sendRequest(searchRequest, RequestCreator.search(), SearchResponse.class, headers) //
+		searchRequest.source().trackTotalHits(true);
+		searchRequest.source().size(0);
+		searchRequest.source().fetchSource(false);
+		return sendRequest(searchRequest, requestCreator.search(), SearchResponse.class, headers) //
 				.map(SearchResponse::getHits) //
 				.map(searchHits -> searchHits.getTotalHits().value) //
 				.next();
@@ -360,11 +411,29 @@ public class DefaultReactiveElasticsearchClient implements ReactiveElasticsearch
 	@Override
 	public Flux<SearchHit> search(HttpHeaders headers, SearchRequest searchRequest) {
 
-		return sendRequest(searchRequest, RequestCreator.search(), SearchResponse.class, headers) //
+		return sendRequest(searchRequest, requestCreator.search(), SearchResponse.class, headers) //
 				.map(SearchResponse::getHits) //
 				.flatMap(Flux::fromIterable);
 	}
 
+	/*
+	 * (non-Javadoc)
+	 * @see org.springframework.data.elasticsearch.client.reactive.ReactiveElasticsearchClient#aggregate(org.springframework.http.HttpHeaders, org.elasticsearch.action.search.SearchRequest)
+	 */
+	@Override
+	public Flux<Aggregation> aggregate(HttpHeaders headers, SearchRequest searchRequest) {
+
+		Assert.notNull(headers, "headers must not be null");
+		Assert.notNull(searchRequest, "searchRequest must not be null");
+
+		searchRequest.source().size(0);
+		searchRequest.source().trackTotalHits(false);
+
+		return sendRequest(searchRequest, requestCreator.search(), SearchResponse.class, headers) //
+				.map(SearchResponse::getAggregations) //
+				.flatMap(Flux::fromIterable);
+	}
+
 	/*
 	 * (non-Javadoc)
 	 * @see org.springframework.data.elasticsearch.client.reactive.ReactiveElasticsearchClient#scroll(org.springframework.http.HttpHeaders, org.elasticsearch.action.search.SearchRequest)
@@ -387,11 +456,11 @@ public class DefaultReactiveElasticsearchClient implements ReactiveElasticsearch
 		Flux<SearchResponse> exchange = outbound.startWith(searchRequest).flatMap(it -> {
 
 			if (it instanceof SearchRequest) {
-				return sendRequest((SearchRequest) it, RequestCreator.search(), SearchResponse.class, headers);
+				return sendRequest((SearchRequest) it, requestCreator.search(), SearchResponse.class, headers);
 			} else if (it instanceof SearchScrollRequest) {
-				return sendRequest((SearchScrollRequest) it, RequestCreator.scroll(), SearchResponse.class, headers);
+				return sendRequest((SearchScrollRequest) it, requestCreator.scroll(), SearchResponse.class, headers);
 			} else if (it instanceof ClearScrollRequest) {
-				return sendRequest((ClearScrollRequest) it, RequestCreator.clearScroll(), ClearScrollResponse.class, headers)
+				return sendRequest((ClearScrollRequest) it, requestCreator.clearScroll(), ClearScrollResponse.class, headers)
 						.flatMap(discard -> Flux.empty());
 			}
 
@@ -444,7 +513,7 @@ public class DefaultReactiveElasticsearchClient implements ReactiveElasticsearch
 		clearScrollRequest.scrollIds(state.getScrollIds());
 
 		// just send the request, resources get cleaned up anyways after scrollTimeout has been reached.
-		return sendRequest(clearScrollRequest, RequestCreator.clearScroll(), ClearScrollResponse.class, headers);
+		return sendRequest(clearScrollRequest, requestCreator.clearScroll(), ClearScrollResponse.class, headers);
 	}
 
 	/*
@@ -454,7 +523,7 @@ public class DefaultReactiveElasticsearchClient implements ReactiveElasticsearch
 	@Override
 	public Mono<BulkByScrollResponse> deleteBy(HttpHeaders headers, DeleteByQueryRequest deleteRequest) {
 
-		return sendRequest(deleteRequest, RequestCreator.deleteByQuery(), BulkByScrollResponse.class, headers) //
+		return sendRequest(deleteRequest, requestCreator.deleteByQuery(), BulkByScrollResponse.class, headers) //
 				.publishNext();
 	}
 
@@ -464,7 +533,7 @@ public class DefaultReactiveElasticsearchClient implements ReactiveElasticsearch
 	 */
 	@Override
 	public Mono<BulkResponse> bulk(HttpHeaders headers, BulkRequest bulkRequest) {
-		return sendRequest(bulkRequest, RequestCreator.bulk(), BulkResponse.class, headers) //
+		return sendRequest(bulkRequest, requestCreator.bulk(), BulkResponse.class, headers) //
 				.publishNext();
 	}
 
@@ -477,7 +546,7 @@ public class DefaultReactiveElasticsearchClient implements ReactiveElasticsearch
 	@Override
 	public Mono<Boolean> existsIndex(HttpHeaders headers, GetIndexRequest request) {
 
-		return sendRequest(request, RequestCreator.indexExists(), RawActionResponse.class, headers) //
+		return sendRequest(request, requestCreator.indexExists(), RawActionResponse.class, headers) //
 				.map(response -> response.statusCode().is2xxSuccessful()) //
 				.next();
 	}
@@ -489,7 +558,7 @@ public class DefaultReactiveElasticsearchClient implements ReactiveElasticsearch
 	@Override
 	public Mono<Void> deleteIndex(HttpHeaders headers, DeleteIndexRequest request) {
 
-		return sendRequest(request, RequestCreator.indexDelete(), AcknowledgedResponse.class, headers) //
+		return sendRequest(request, requestCreator.indexDelete(), AcknowledgedResponse.class, headers) //
 				.then();
 	}
 
@@ -500,7 +569,7 @@ public class DefaultReactiveElasticsearchClient implements ReactiveElasticsearch
 	@Override
 	public Mono<Void> createIndex(HttpHeaders headers, CreateIndexRequest createIndexRequest) {
 
-		return sendRequest(createIndexRequest, RequestCreator.indexCreate(), AcknowledgedResponse.class, headers) //
+		return sendRequest(createIndexRequest, requestCreator.indexCreate(), AcknowledgedResponse.class, headers) //
 				.then();
 	}
 
@@ -511,7 +580,7 @@ public class DefaultReactiveElasticsearchClient implements ReactiveElasticsearch
 	@Override
 	public Mono<Void> openIndex(HttpHeaders headers, OpenIndexRequest request) {
 
-		return sendRequest(request, RequestCreator.indexOpen(), AcknowledgedResponse.class, headers) //
+		return sendRequest(request, requestCreator.indexOpen(), AcknowledgedResponse.class, headers) //
 				.then();
 	}
 
@@ -522,7 +591,7 @@ public class DefaultReactiveElasticsearchClient implements ReactiveElasticsearch
 	@Override
 	public Mono<Void> closeIndex(HttpHeaders headers, CloseIndexRequest closeIndexRequest) {
 
-		return sendRequest(closeIndexRequest, RequestCreator.indexClose(), AcknowledgedResponse.class, headers) //
+		return sendRequest(closeIndexRequest, requestCreator.indexClose(), AcknowledgedResponse.class, headers) //
 				.then();
 	}
 
@@ -533,7 +602,7 @@ public class DefaultReactiveElasticsearchClient implements ReactiveElasticsearch
 	@Override
 	public Mono<Void> refreshIndex(HttpHeaders headers, RefreshRequest refreshRequest) {
 
-		return sendRequest(refreshRequest, RequestCreator.indexRefresh(), RefreshResponse.class, headers) //
+		return sendRequest(refreshRequest, requestCreator.indexRefresh(), RefreshResponse.class, headers) //
 				.then();
 	}
 
@@ -544,7 +613,7 @@ public class DefaultReactiveElasticsearchClient implements ReactiveElasticsearch
 	@Override
 	public Mono<Void> updateMapping(HttpHeaders headers, PutMappingRequest putMappingRequest) {
 
-		return sendRequest(putMappingRequest, RequestCreator.putMapping(), AcknowledgedResponse.class, headers) //
+		return sendRequest(putMappingRequest, requestCreator.putMapping(), AcknowledgedResponse.class, headers) //
 				.then();
 	}
 
@@ -555,7 +624,7 @@ public class DefaultReactiveElasticsearchClient implements ReactiveElasticsearch
 	@Override
 	public Mono<Void> flushIndex(HttpHeaders headers, FlushRequest flushRequest) {
 
-		return sendRequest(flushRequest, RequestCreator.flushIndex(), FlushResponse.class, headers) //
+		return sendRequest(flushRequest, requestCreator.flushIndex(), FlushResponse.class, headers) //
 				.then();
 	}
 
@@ -638,6 +707,12 @@ public class DefaultReactiveElasticsearchClient implements ReactiveElasticsearch
 							request.getOptions().getHeaders().forEach(it -> theHeaders.add(it.getName(), it.getValue()));
 						}
 					}
+
+					// plus the ones from the supplier
+					HttpHeaders suppliedHeaders = headersSupplier.get();
+					if (suppliedHeaders != null && suppliedHeaders != HttpHeaders.EMPTY) {
+						theHeaders.addAll(suppliedHeaders);
+					}
 				});
 
 		if (request.getEntity() != null) {
@@ -648,7 +723,7 @@ public class DefaultReactiveElasticsearchClient implements ReactiveElasticsearch
 					body::get);
 
 			requestBodySpec.contentType(MediaType.valueOf(request.getEntity().getContentType().getValue()));
-			requestBodySpec.body(Mono.fromSupplier(body::get), String.class);
+			requestBodySpec.body(Mono.fromSupplier(body), String.class);
 		} else {
 			ClientLogger.logRequest(logId, request.getMethod().toUpperCase(), request.getEndpoint(), request.getParameters());
 		}
@@ -685,6 +760,12 @@ public class DefaultReactiveElasticsearchClient implements ReactiveElasticsearch
 			return handleServerError(request, response);
 		}
 
+		if (response.statusCode().is4xxClientError()) {
+
+			ClientLogger.logRawResponse(logId, response.statusCode());
+			return handleClientError(logId, request, response, responseType);
+		}
+
 		return response.body(BodyExtractors.toMono(byte[].class)) //
 				.map(it -> new String(it, StandardCharsets.UTF_8)) //
 				.doOnNext(it -> ClientLogger.logResponse(logId, response.statusCode(), it)) //
@@ -715,122 +796,106 @@ public class DefaultReactiveElasticsearchClient implements ReactiveElasticsearch
 	}
 
 	private static XContentParser createParser(String mediaType, String content) throws IOException {
-
 		return XContentType.fromMediaTypeOrFormat(mediaType) //
 				.xContent() //
-				.createParser(NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, content);
+				.createParser(new NamedXContentRegistry(NamedXContents.getDefaultNamedXContents()),
+						DeprecationHandler.THROW_UNSUPPORTED_OPERATION, content);
 	}
 
-	private static <T> Publisher<? extends T> handleServerError(Request request, ClientResponse response) {
+	private <T> Publisher<? extends T> handleServerError(Request request, ClientResponse response) {
 
-		return Mono.error(
-				new HttpServerErrorException(response.statusCode(), String.format("%s request to %s returned error code %s.",
-						request.getMethod(), request.getEndpoint(), response.statusCode().value())));
+		int statusCode = response.statusCode().value();
+		RestStatus status = RestStatus.fromCode(statusCode);
+		String mediaType = response.headers().contentType().map(MediaType::toString).orElse(XContentType.JSON.mediaType());
+
+		return response.body(BodyExtractors.toMono(byte[].class)) //
+				.switchIfEmpty(Mono.error(
+						new ElasticsearchStatusException(String.format("%s request to %s returned error code %s and no body.",
+								request.getMethod(), request.getEndpoint(), statusCode), status)))
+				.map(bytes -> new String(bytes, StandardCharsets.UTF_8)) //
+				.flatMap(content -> contentOrError(content, mediaType, status))
+				.flatMap(unused -> Mono
+						.error(new ElasticsearchStatusException(String.format("%s request to %s returned error code %s.",
+								request.getMethod(), request.getEndpoint(), statusCode), status)));
 	}
 
-	static class RequestCreator {
+	private <T> Publisher<? extends T> handleClientError(String logId, Request request, ClientResponse response,
+			Class<T> responseType) {
 
-		static Function<SearchRequest, Request> search() {
-			return RequestConverters::search;
+		int statusCode = response.statusCode().value();
+		RestStatus status = RestStatus.fromCode(statusCode);
+		String mediaType = response.headers().contentType().map(MediaType::toString).orElse(XContentType.JSON.mediaType());
+
+		return response.body(BodyExtractors.toMono(byte[].class)) //
+				.map(bytes -> new String(bytes, StandardCharsets.UTF_8)) //
+				.flatMap(content -> contentOrError(content, mediaType, status)) //
+				.doOnNext(content -> ClientLogger.logResponse(logId, response.statusCode(), content)) //
+				.flatMap(content -> doDecode(response, responseType, content));
+	}
+
+	// region ElasticsearchException helper
+	/**
+	 * checks if the given content body contains an {@link ElasticsearchException}, if yes it is returned in a Mono.error.
+	 * Otherwise the content is returned in the Mono
+	 *
+	 * @param content the content to analyze
+	 * @param mediaType the returned media type
+	 * @param status the response status
+	 * @return a Mono with the content or an Mono.error
+	 */
+	private static Mono<String> contentOrError(String content, String mediaType, RestStatus status) {
+
+		ElasticsearchException exception = getElasticsearchException(content, mediaType, status);
+
+		if (exception != null) {
+			StringBuilder sb = new StringBuilder();
+			buildExceptionMessages(sb, exception);
+			return Mono.error(new ElasticsearchStatusException(sb.toString(), status, exception));
 		}
 
-		static Function<SearchScrollRequest, Request> scroll() {
-			return RequestConverters::searchScroll;
-		}
+		return Mono.just(content);
+	}
 
-		static Function<ClearScrollRequest, Request> clearScroll() {
-			return RequestConverters::clearScroll;
-		}
+	/**
+	 * tries to parse an {@link ElasticsearchException} from the given body content
+	 *
+	 * @param content the content to analyse
+	 * @param mediaType the type of the body content
+	 * @return an {@link ElasticsearchException} or {@literal null}.
+	 */
+	@Nullable
+	private static ElasticsearchException getElasticsearchException(String content, String mediaType, RestStatus status) {
 
-		static Function<IndexRequest, Request> index() {
-			return RequestConverters::index;
-		}
+		try {
+			XContentParser parser = createParser(mediaType, content);
+			// we have a JSON object with an error and a status field
+			XContentParser.Token token = parser.nextToken(); // Skip START_OBJECT
 
-		static Function<GetRequest, Request> get() {
-			return RequestConverters::get;
-		}
+			do {
+				token = parser.nextToken();
 
-		static Function<MainRequest, Request> ping() {
-			return (request) -> RequestConverters.ping();
-		}
-
-		static Function<MainRequest, Request> info() {
-			return (request) -> RequestConverters.info();
-		}
-
-		static Function<MultiGetRequest, Request> multiGet() {
-			return RequestConverters::multiGet;
-		}
-
-		static Function<GetRequest, Request> exists() {
-			return RequestConverters::exists;
-		}
-
-		static Function<UpdateRequest, Request> update() {
-			return RequestConverters::update;
-		}
-
-		static Function<DeleteRequest, Request> delete() {
-			return RequestConverters::delete;
-		}
-
-		static Function<DeleteByQueryRequest, Request> deleteByQuery() {
-
-			return request -> RequestConverters.deleteByQuery(request);
-		}
-
-		static Function<BulkRequest, Request> bulk() {
-
-			return request -> {
-
-				try {
-					return RequestConverters.bulk(request);
-				} catch (IOException e) {
-					throw new ElasticsearchException("Could not parse request", e);
+				if (parser.currentName().equals("error")) {
+					return ElasticsearchException.failureFromXContent(parser);
 				}
-			};
+			} while (token == XContentParser.Token.FIELD_NAME);
+
+			return null;
+		} catch (IOException e) {
+			return new ElasticsearchStatusException(content, status);
 		}
-
-		// --> INDICES
-
-		static Function<GetIndexRequest, Request> indexExists() {
-			return RequestConverters::indexExists;
-		}
-
-		static Function<DeleteIndexRequest, Request> indexDelete() {
-			return RequestConverters::indexDelete;
-		}
-
-		static Function<CreateIndexRequest, Request> indexCreate() {
-			return RequestConverters::indexCreate;
-		}
-
-		static Function<OpenIndexRequest, Request> indexOpen() {
-			return RequestConverters::indexOpen;
-		}
-
-		static Function<CloseIndexRequest, Request> indexClose() {
-			return RequestConverters::indexClose;
-		}
-
-		static Function<RefreshRequest, Request> indexRefresh() {
-			return RequestConverters::indexRefresh;
-		}
-
-		static Function<PutMappingRequest, Request> putMapping() {
-			return RequestConverters::putMapping;
-		}
-
-		static Function<FlushRequest, Request> flushIndex() {
-			return RequestConverters::flushIndex;
-		}
-
-		static Function<CountRequest, Request> count() {
-			return RequestConverters::count;
-		}
-
 	}
 
+	private static void buildExceptionMessages(StringBuilder sb, Throwable t) {
+
+		sb.append(t.getMessage());
+		for (Throwable throwable : t.getSuppressed()) {
+			sb.append(", ");
+			buildExceptionMessages(sb, throwable);
+		}
+	}
+	// endregion
+
+	// region internal classes
 	/**
 	 * Reactive client {@link ReactiveElasticsearchClient.Status} implementation.
 	 *
@@ -854,41 +919,5 @@ public class DefaultReactiveElasticsearchClient implements ReactiveElasticsearch
 		}
 	}
 
-	/**
-	 * Mutable state object holding scrollId to be used for {@link SearchScrollRequest#scroll(Scroll)}
-	 *
-	 * @author Christoph Strobl
-	 * @since 3.2
-	 */
-	private static class ScrollState {
-
-		private final Object lock = new Object();
-
-		private final List<String> pastIds = new ArrayList<>(1);
-		@Nullable private String scrollId;
-
-		@Nullable
-		String getScrollId() {
-			return scrollId;
-		}
-
-		List<String> getScrollIds() {
-
-			synchronized (lock) {
-				return Collections.unmodifiableList(new ArrayList<>(pastIds));
-			}
-		}
-
-		void updateScrollId(String scrollId) {
-
-			if (StringUtils.hasText(scrollId)) {
-
-				synchronized (lock) {
-
-					this.scrollId = scrollId;
-					pastIds.add(scrollId);
-				}
-			}
-		}
-	}
+	// endregion
 }

src/main/java/org/springframework/data/elasticsearch/client/reactive/DefaultRequestCreator.java

@@ -0,0 +1,7 @@
+package org.springframework.data.elasticsearch.client.reactive;
+
+/**
+ * @author Roman Puchkovskiy
+ * @since 4.0
+ */
+class DefaultRequestCreator implements RequestCreator {}

src/main/java/org/springframework/data/elasticsearch/client/reactive/DefaultWebClientProvider.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2018-2020 the original author or authors.
+ * Copyright 2018-2021 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -27,6 +27,7 @@ import org.springframework.lang.Nullable;
 import org.springframework.util.Assert;
 import org.springframework.web.reactive.function.client.WebClient;
 import org.springframework.web.reactive.function.client.WebClient.Builder;
+import org.springframework.web.util.DefaultUriBuilderFactory;
 
 /**
  * Default {@link WebClientProvider} that uses cached {@link WebClient} instances per {@code hostAndPort}.
@@ -156,7 +157,16 @@ class DefaultWebClientProvider implements WebClientProvider {
 
 		String baseUrl = String.format("%s://%s:%d%s", this.scheme, socketAddress.getHostString(), socketAddress.getPort(),
 				pathPrefix == null ? "" : '/' + pathPrefix);
-		WebClient webClient = builder.baseUrl(baseUrl).filter((request, next) -> next.exchange(request).doOnError(errorListener)).build();
+
+		DefaultUriBuilderFactory uriBuilderFactory = new DefaultUriBuilderFactory(baseUrl);
+		// the template will already be encoded by the RequestConverters methods
+		uriBuilderFactory.setEncodingMode(DefaultUriBuilderFactory.EncodingMode.VALUES_ONLY);
+		builder.uriBuilderFactory(uriBuilderFactory); //
+
+		WebClient webClient = builder //
+				.filter((request, next) -> next.exchange(request) //
+						.doOnError(errorListener)) //
+				.build(); //
 		return webClientConfigurer.apply(webClient);
 	}
 }

src/main/java/org/springframework/data/elasticsearch/client/reactive/HostProvider.java

@@ -20,9 +20,11 @@ import reactor.core.publisher.Mono;
 import java.net.InetSocketAddress;
 import java.util.Collections;
 import java.util.Set;
+import java.util.function.Supplier;
 
 import org.springframework.data.elasticsearch.client.ElasticsearchHost;
 import org.springframework.data.elasticsearch.client.NoReachableHostException;
+import org.springframework.http.HttpHeaders;
 import org.springframework.util.Assert;
 import org.springframework.web.reactive.function.client.WebClient;
 
@@ -40,18 +42,20 @@ public interface HostProvider {
 	 * Create a new {@link HostProvider} best suited for the given {@link WebClientProvider} and number of hosts.
 	 *
 	 * @param clientProvider must not be {@literal null} .
+	 * @param headersSupplier  to supply custom headers, must not be {@literal null}
 	 * @param endpoints must not be {@literal null} nor empty.
 	 * @return new instance of {@link HostProvider}.
 	 */
-	static HostProvider provider(WebClientProvider clientProvider, InetSocketAddress... endpoints) {
+	static HostProvider provider(WebClientProvider clientProvider, Supplier<HttpHeaders> headersSupplier,
+			InetSocketAddress... endpoints) {
 
 		Assert.notNull(clientProvider, "WebClientProvider must not be null");
 		Assert.notEmpty(endpoints, "Please provide at least one endpoint to connect to.");
 
 		if (endpoints.length == 1) {
-			return new SingleNodeHostProvider(clientProvider, endpoints[0]);
+			return new SingleNodeHostProvider(clientProvider, headersSupplier, endpoints[0]);
 		} else {
-			return new MultiNodeHostProvider(clientProvider, endpoints);
+			return new MultiNodeHostProvider(clientProvider,headersSupplier, endpoints);
 		}
 	}
 

src/main/java/org/springframework/data/elasticsearch/client/reactive/MultiNodeHostProvider.java

@@ -15,6 +15,7 @@
  */
 package org.springframework.data.elasticsearch.client.reactive;
 
+import org.springframework.http.HttpHeaders;
 import reactor.core.publisher.Flux;
 import reactor.core.publisher.Mono;
 import reactor.util.function.Tuple2;
@@ -27,6 +28,7 @@ import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.function.Supplier;
 
 import org.springframework.data.elasticsearch.client.ElasticsearchHost;
 import org.springframework.data.elasticsearch.client.ElasticsearchHost.State;
@@ -45,11 +47,13 @@ import org.springframework.web.reactive.function.client.WebClient;
 class MultiNodeHostProvider implements HostProvider {
 
 	private final WebClientProvider clientProvider;
+	private final Supplier<HttpHeaders> headersSupplier;
 	private final Map<InetSocketAddress, ElasticsearchHost> hosts;
 
-	MultiNodeHostProvider(WebClientProvider clientProvider, InetSocketAddress... endpoints) {
+	MultiNodeHostProvider(WebClientProvider clientProvider, Supplier<HttpHeaders> headersSupplier, InetSocketAddress... endpoints) {
 
 		this.clientProvider = clientProvider;
+		this.headersSupplier = headersSupplier;
 		this.hosts = new ConcurrentHashMap<>();
 		for (InetSocketAddress endpoint : endpoints) {
 			this.hosts.put(endpoint, new ElasticsearchHost(endpoint, State.UNKNOWN));
@@ -133,8 +137,9 @@ class MultiNodeHostProvider implements HostProvider {
 				.flatMap(host -> {
 
 					Mono<ClientResponse> exchange = createWebClient(host) //
-							.head().uri("/").exchange().doOnError(throwable -> {
-
+							.head().uri("/") //
+							.headers(httpHeaders -> httpHeaders.addAll(headersSupplier.get())) //
+							.exchange().doOnError(throwable -> {
 								hosts.put(host, new ElasticsearchHost(host, State.OFFLINE));
 								clientProvider.getErrorListener().accept(throwable);
 							});

src/main/java/org/springframework/data/elasticsearch/client/reactive/ReactiveElasticsearchClient.java

@@ -47,9 +47,11 @@ import org.elasticsearch.index.get.GetResult;
 import org.elasticsearch.index.reindex.BulkByScrollResponse;
 import org.elasticsearch.index.reindex.DeleteByQueryRequest;
 import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.search.aggregations.Aggregation;
 import org.springframework.data.elasticsearch.client.ClientConfiguration;
 import org.springframework.data.elasticsearch.client.ElasticsearchHost;
 import org.springframework.http.HttpHeaders;
+import org.springframework.util.Assert;
 import org.springframework.util.CollectionUtils;
 import org.springframework.web.reactive.function.client.ClientResponse;
 import org.springframework.web.reactive.function.client.WebClient;
@@ -411,6 +413,50 @@ public interface ReactiveElasticsearchClient {
 	 */
 	Flux<SearchHit> search(HttpHeaders headers, SearchRequest searchRequest);
 
+	/**
+	 * Execute the given {@link SearchRequest} with aggregations against the {@literal search} API.
+	 *
+	 * @param consumer
+	 *     never {@literal null}.
+	 * @return the {@link Flux} emitting {@link Aggregation} one by one.
+	 * @see <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/search-search.html">Search API on
+	 * elastic.co</a>
+	 * @since 4.0
+	 */
+	default Flux<Aggregation> aggregate(Consumer<SearchRequest> consumer) {
+
+		Assert.notNull(consumer, "consumer must not be null");
+
+		SearchRequest request = new SearchRequest();
+		consumer.accept(request);
+		return aggregate(request);
+	}
+
+	/**
+	 * Execute the given {@link SearchRequest} with aggregations against the {@literal search} API.
+	 *
+	 * @param searchRequest must not be {@literal null}.
+	 * @see <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/search-search.html">Search API on
+	 *      elastic.co</a>
+	 * @return the {@link Flux} emitting {@link Aggregation} one by one.
+	 * @since 4.0
+	 */
+	default Flux<Aggregation> aggregate(SearchRequest searchRequest) {
+		return aggregate(HttpHeaders.EMPTY, searchRequest);
+	}
+
+	/**
+	 * Execute the given {@link SearchRequest} with aggregations against the {@literal search} API.
+	 *
+	 * @param headers Use {@link HttpHeaders} to provide eg. authentication data. Must not be {@literal null}.
+	 * @param searchRequest must not be {@literal null}.
+	 * @see <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/search-search.html">Search API on
+	 *      elastic.co</a>
+	 * @return the {@link Flux} emitting {@link Aggregation} one by one.
+	 * @since 4.0
+	 */
+	Flux<Aggregation> aggregate(HttpHeaders headers, SearchRequest searchRequest);
+
 	/**
 	 * Execute the given {@link SearchRequest} against the {@literal search scroll} API.
 	 *

src/main/java/org/springframework/data/elasticsearch/client/reactive/ReactiveRestClients.java

@@ -24,6 +24,7 @@ import org.springframework.util.Assert;
  *
  * @author Christoph Strobl
  * @author Mark Paluch
+ * @author Roman Puchkovskiy
  * @since 3.2
  */
 public final class ReactiveRestClients {
@@ -33,6 +34,8 @@ public final class ReactiveRestClients {
 	/**
 	 * Start here to create a new client tailored to your needs.
 	 *
+	 * @param clientConfiguration client configuration to use for building {@link ReactiveElasticsearchClient}; must not
+	 *          be {@literal null}.
 	 * @return new instance of {@link ReactiveElasticsearchClient}.
 	 */
 	public static ReactiveElasticsearchClient create(ClientConfiguration clientConfiguration) {
@@ -41,4 +44,21 @@ public final class ReactiveRestClients {
 
 		return DefaultReactiveElasticsearchClient.create(clientConfiguration);
 	}
+
+	/**
+	 * Start here to create a new client tailored to your needs.
+	 *
+	 * @param clientConfiguration client configuration to use for building {@link ReactiveElasticsearchClient}; must not
+	 *          be {@literal null}.
+	 * @param requestCreator request creator to use in the client; must not be {@literal null}.
+	 * @return new instance of {@link ReactiveElasticsearchClient}.
+	 */
+	public static ReactiveElasticsearchClient create(ClientConfiguration clientConfiguration,
+			RequestCreator requestCreator) {
+
+		Assert.notNull(clientConfiguration, "ClientConfiguration must not be null!");
+		Assert.notNull(requestCreator, "RequestCreator must not be null!");
+
+		return DefaultReactiveElasticsearchClient.create(clientConfiguration, requestCreator);
+	}
 }

src/main/java/org/springframework/data/elasticsearch/client/reactive/RequestCreator.java

@@ -0,0 +1,134 @@
+package org.springframework.data.elasticsearch.client.reactive;
+
+import java.io.IOException;
+import java.util.function.Function;
+
+import org.elasticsearch.action.admin.indices.close.CloseIndexRequest;
+import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
+import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
+import org.elasticsearch.action.admin.indices.flush.FlushRequest;
+import org.elasticsearch.action.admin.indices.get.GetIndexRequest;
+import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
+import org.elasticsearch.action.admin.indices.open.OpenIndexRequest;
+import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
+import org.elasticsearch.action.bulk.BulkRequest;
+import org.elasticsearch.action.delete.DeleteRequest;
+import org.elasticsearch.action.get.GetRequest;
+import org.elasticsearch.action.get.MultiGetRequest;
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.action.main.MainRequest;
+import org.elasticsearch.action.search.ClearScrollRequest;
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.action.search.SearchScrollRequest;
+import org.elasticsearch.action.update.UpdateRequest;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.core.CountRequest;
+import org.elasticsearch.index.reindex.DeleteByQueryRequest;
+import org.springframework.data.elasticsearch.ElasticsearchException;
+import org.springframework.data.elasticsearch.client.util.RequestConverters;
+
+/**
+ * @author Roman Puchkovskiy
+ * @since 4.0
+ */
+public interface RequestCreator {
+
+	default Function<SearchRequest, Request> search() {
+		return RequestConverters::search;
+	}
+
+	default Function<SearchScrollRequest, Request> scroll() {
+		return RequestConverters::searchScroll;
+	}
+
+	default Function<ClearScrollRequest, Request> clearScroll() {
+		return RequestConverters::clearScroll;
+	}
+
+	default Function<IndexRequest, Request> index() {
+		return RequestConverters::index;
+	}
+
+	default Function<GetRequest, Request> get() {
+		return RequestConverters::get;
+	}
+
+	default Function<MainRequest, Request> ping() {
+		return (request) -> RequestConverters.ping();
+	}
+
+	default Function<MainRequest, Request> info() {
+		return (request) -> RequestConverters.info();
+	}
+
+	default Function<MultiGetRequest, Request> multiGet() {
+		return RequestConverters::multiGet;
+	}
+
+	default Function<GetRequest, Request> exists() {
+		return RequestConverters::exists;
+	}
+
+	default Function<UpdateRequest, Request> update() {
+		return RequestConverters::update;
+	}
+
+	default Function<DeleteRequest, Request> delete() {
+		return RequestConverters::delete;
+	}
+
+	default Function<DeleteByQueryRequest, Request> deleteByQuery() {
+		return RequestConverters::deleteByQuery;
+	}
+
+	default Function<BulkRequest, Request> bulk() {
+
+		return request -> {
+
+			try {
+				return RequestConverters.bulk(request);
+			} catch (IOException e) {
+				throw new ElasticsearchException("Could not parse request", e);
+			}
+		};
+	}
+
+	// --> INDICES
+
+	default Function<GetIndexRequest, Request> indexExists() {
+		return RequestConverters::indexExists;
+	}
+
+	default Function<DeleteIndexRequest, Request> indexDelete() {
+		return RequestConverters::indexDelete;
+	}
+
+	default Function<CreateIndexRequest, Request> indexCreate() {
+		return RequestConverters::indexCreate;
+	}
+
+	default Function<OpenIndexRequest, Request> indexOpen() {
+		return RequestConverters::indexOpen;
+	}
+
+	default Function<CloseIndexRequest, Request> indexClose() {
+		return RequestConverters::indexClose;
+	}
+
+	default Function<RefreshRequest, Request> indexRefresh() {
+		return RequestConverters::indexRefresh;
+	}
+
+	default Function<PutMappingRequest, Request> putMapping() {
+		return RequestConverters::putMapping;
+	}
+
+	default Function<FlushRequest, Request> flushIndex() {
+		return RequestConverters::flushIndex;
+	}
+
+	default Function<CountRequest, Request> count() {
+		return RequestConverters::count;
+	}
+
+}

src/main/java/org/springframework/data/elasticsearch/client/reactive/SingleNodeHostProvider.java

@@ -15,10 +15,12 @@
  */
 package org.springframework.data.elasticsearch.client.reactive;
 
+import org.springframework.http.HttpHeaders;
 import reactor.core.publisher.Mono;
 
 import java.net.InetSocketAddress;
 import java.util.Collections;
+import java.util.function.Supplier;
 
 import org.springframework.data.elasticsearch.client.ElasticsearchHost;
 import org.springframework.data.elasticsearch.client.ElasticsearchHost.State;
@@ -35,12 +37,14 @@ import org.springframework.web.reactive.function.client.WebClient;
 class SingleNodeHostProvider implements HostProvider {
 
 	private final WebClientProvider clientProvider;
+	private final Supplier<HttpHeaders> headersSupplier;
 	private final InetSocketAddress endpoint;
 	private volatile ElasticsearchHost state;
 
-	SingleNodeHostProvider(WebClientProvider clientProvider, InetSocketAddress endpoint) {
+	SingleNodeHostProvider(WebClientProvider clientProvider, Supplier<HttpHeaders> headersSupplier, InetSocketAddress endpoint) {
 
 		this.clientProvider = clientProvider;
+		this.headersSupplier = headersSupplier;
 		this.endpoint = endpoint;
 		this.state = new ElasticsearchHost(this.endpoint, State.UNKNOWN);
 	}
@@ -53,9 +57,10 @@ class SingleNodeHostProvider implements HostProvider {
 	public Mono<ClusterInformation> clusterInfo() {
 
 		return createWebClient(endpoint) //
-				.head().uri("/").exchange() //
+				.head().uri("/")
+				.headers(httpHeaders -> httpHeaders.addAll(headersSupplier.get())) //
+				.exchange() //
 				.flatMap(it -> {
-
 					if (it.statusCode().isError()) {
 						state = ElasticsearchHost.offline(endpoint);
 					} else {

src/main/java/org/springframework/data/elasticsearch/client/util/NamedXContents.java

@@ -0,0 +1,169 @@
+/*
+ * Copyright 2020 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.springframework.data.elasticsearch.client.util;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.xcontent.ContextParser;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.search.aggregations.Aggregation;
+import org.elasticsearch.search.aggregations.bucket.adjacency.AdjacencyMatrixAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.adjacency.ParsedAdjacencyMatrix;
+import org.elasticsearch.search.aggregations.bucket.composite.CompositeAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.composite.ParsedComposite;
+import org.elasticsearch.search.aggregations.bucket.filter.FilterAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.filter.FiltersAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.filter.ParsedFilter;
+import org.elasticsearch.search.aggregations.bucket.filter.ParsedFilters;
+import org.elasticsearch.search.aggregations.bucket.geogrid.GeoHashGridAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.geogrid.ParsedGeoHashGrid;
+import org.elasticsearch.search.aggregations.bucket.geogrid.ParsedGeoTileGrid;
+import org.elasticsearch.search.aggregations.bucket.global.GlobalAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.global.ParsedGlobal;
+import org.elasticsearch.search.aggregations.bucket.histogram.AutoDateHistogramAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.histogram.HistogramAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.histogram.ParsedAutoDateHistogram;
+import org.elasticsearch.search.aggregations.bucket.histogram.ParsedDateHistogram;
+import org.elasticsearch.search.aggregations.bucket.histogram.ParsedHistogram;
+import org.elasticsearch.search.aggregations.bucket.missing.MissingAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.missing.ParsedMissing;
+import org.elasticsearch.search.aggregations.bucket.nested.NestedAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.nested.ParsedNested;
+import org.elasticsearch.search.aggregations.bucket.nested.ParsedReverseNested;
+import org.elasticsearch.search.aggregations.bucket.nested.ReverseNestedAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.range.DateRangeAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.range.GeoDistanceAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.range.IpRangeAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.range.ParsedBinaryRange;
+import org.elasticsearch.search.aggregations.bucket.range.ParsedDateRange;
+import org.elasticsearch.search.aggregations.bucket.range.ParsedGeoDistance;
+import org.elasticsearch.search.aggregations.bucket.range.ParsedRange;
+import org.elasticsearch.search.aggregations.bucket.range.RangeAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.sampler.InternalSampler;
+import org.elasticsearch.search.aggregations.bucket.sampler.ParsedSampler;
+import org.elasticsearch.search.aggregations.bucket.significant.ParsedSignificantLongTerms;
+import org.elasticsearch.search.aggregations.bucket.significant.ParsedSignificantStringTerms;
+import org.elasticsearch.search.aggregations.bucket.significant.SignificantLongTerms;
+import org.elasticsearch.search.aggregations.bucket.significant.SignificantStringTerms;
+import org.elasticsearch.search.aggregations.bucket.terms.DoubleTerms;
+import org.elasticsearch.search.aggregations.bucket.terms.LongTerms;
+import org.elasticsearch.search.aggregations.bucket.terms.ParsedDoubleTerms;
+import org.elasticsearch.search.aggregations.bucket.terms.ParsedLongTerms;
+import org.elasticsearch.search.aggregations.bucket.terms.ParsedStringTerms;
+import org.elasticsearch.search.aggregations.bucket.terms.StringTerms;
+import org.elasticsearch.search.aggregations.metrics.*;
+import org.elasticsearch.search.aggregations.pipeline.*;
+import org.elasticsearch.search.suggest.Suggest;
+import org.elasticsearch.search.suggest.completion.CompletionSuggestion;
+import org.elasticsearch.search.suggest.completion.CompletionSuggestionBuilder;
+import org.elasticsearch.search.suggest.phrase.PhraseSuggestion;
+import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder;
+import org.elasticsearch.search.suggest.term.TermSuggestion;
+import org.elasticsearch.search.suggest.term.TermSuggestionBuilder;
+import org.springframework.data.elasticsearch.client.reactive.ReactiveElasticsearchClient;
+
+/**
+ * <p>
+ * Original implementation source {@link org.elasticsearch.client.RestHighLevelClient#getDefaultNamedXContents()} by
+ * {@literal Elasticsearch} (<a href="https://www.elastic.co">https://www.elastic.co</a>) licensed under the Apache
+ * License, Version 2.0.
+ * </p>
+ * Modified for usage with {@link ReactiveElasticsearchClient}.
+ * <p>
+ * Only intended for internal use.
+ *
+ * @author Russell Parry
+ * @since 4.0
+ */
+public class NamedXContents {
+
+	private NamedXContents() {
+		// contains only utility methods
+	}
+
+	public static List<NamedXContentRegistry.Entry> getDefaultNamedXContents() {
+		Map<String, ContextParser<Object, ? extends Aggregation>> map = new HashMap<>();
+		map.put(CardinalityAggregationBuilder.NAME, (p, c) -> ParsedCardinality.fromXContent(p, (String) c));
+		map.put(InternalHDRPercentiles.NAME, (p, c) -> ParsedHDRPercentiles.fromXContent(p, (String) c));
+		map.put(InternalHDRPercentileRanks.NAME, (p, c) -> ParsedHDRPercentileRanks.fromXContent(p, (String) c));
+		map.put(InternalTDigestPercentiles.NAME, (p, c) -> ParsedTDigestPercentiles.fromXContent(p, (String) c));
+		map.put(InternalTDigestPercentileRanks.NAME, (p, c) -> ParsedTDigestPercentileRanks.fromXContent(p, (String) c));
+		map.put(PercentilesBucketPipelineAggregationBuilder.NAME,
+				(p, c) -> ParsedPercentilesBucket.fromXContent(p, (String) c));
+		map.put(MedianAbsoluteDeviationAggregationBuilder.NAME,
+				(p, c) -> ParsedMedianAbsoluteDeviation.fromXContent(p, (String) c));
+		map.put(MinAggregationBuilder.NAME, (p, c) -> ParsedMin.fromXContent(p, (String) c));
+		map.put(MaxAggregationBuilder.NAME, (p, c) -> ParsedMax.fromXContent(p, (String) c));
+		map.put(SumAggregationBuilder.NAME, (p, c) -> ParsedSum.fromXContent(p, (String) c));
+		map.put(AvgAggregationBuilder.NAME, (p, c) -> ParsedAvg.fromXContent(p, (String) c));
+		map.put(WeightedAvgAggregationBuilder.NAME, (p, c) -> ParsedWeightedAvg.fromXContent(p, (String) c));
+		map.put(ValueCountAggregationBuilder.NAME, (p, c) -> ParsedValueCount.fromXContent(p, (String) c));
+		map.put(InternalSimpleValue.NAME, (p, c) -> ParsedSimpleValue.fromXContent(p, (String) c));
+		map.put(DerivativePipelineAggregationBuilder.NAME, (p, c) -> ParsedDerivative.fromXContent(p, (String) c));
+		map.put(InternalBucketMetricValue.NAME, (p, c) -> ParsedBucketMetricValue.fromXContent(p, (String) c));
+		map.put(StatsAggregationBuilder.NAME, (p, c) -> ParsedStats.fromXContent(p, (String) c));
+		map.put(StatsBucketPipelineAggregationBuilder.NAME, (p, c) -> ParsedStatsBucket.fromXContent(p, (String) c));
+		map.put(ExtendedStatsAggregationBuilder.NAME, (p, c) -> ParsedExtendedStats.fromXContent(p, (String) c));
+		map.put(ExtendedStatsBucketPipelineAggregationBuilder.NAME,
+				(p, c) -> ParsedExtendedStatsBucket.fromXContent(p, (String) c));
+		map.put(GeoBoundsAggregationBuilder.NAME, (p, c) -> ParsedGeoBounds.fromXContent(p, (String) c));
+		map.put(GeoCentroidAggregationBuilder.NAME, (p, c) -> ParsedGeoCentroid.fromXContent(p, (String) c));
+		map.put(HistogramAggregationBuilder.NAME, (p, c) -> ParsedHistogram.fromXContent(p, (String) c));
+		map.put(DateHistogramAggregationBuilder.NAME, (p, c) -> ParsedDateHistogram.fromXContent(p, (String) c));
+		map.put(AutoDateHistogramAggregationBuilder.NAME, (p, c) -> ParsedAutoDateHistogram.fromXContent(p, (String) c));
+		map.put(StringTerms.NAME, (p, c) -> ParsedStringTerms.fromXContent(p, (String) c));
+		map.put(LongTerms.NAME, (p, c) -> ParsedLongTerms.fromXContent(p, (String) c));
+		map.put(DoubleTerms.NAME, (p, c) -> ParsedDoubleTerms.fromXContent(p, (String) c));
+		map.put(MissingAggregationBuilder.NAME, (p, c) -> ParsedMissing.fromXContent(p, (String) c));
+		map.put(NestedAggregationBuilder.NAME, (p, c) -> ParsedNested.fromXContent(p, (String) c));
+		map.put(ReverseNestedAggregationBuilder.NAME, (p, c) -> ParsedReverseNested.fromXContent(p, (String) c));
+		map.put(GlobalAggregationBuilder.NAME, (p, c) -> ParsedGlobal.fromXContent(p, (String) c));
+		map.put(FilterAggregationBuilder.NAME, (p, c) -> ParsedFilter.fromXContent(p, (String) c));
+		map.put(InternalSampler.PARSER_NAME, (p, c) -> ParsedSampler.fromXContent(p, (String) c));
+		map.put(GeoHashGridAggregationBuilder.NAME, (p, c) -> ParsedGeoHashGrid.fromXContent(p, (String) c));
+		map.put(GeoTileGridAggregationBuilder.NAME, (p, c) -> ParsedGeoTileGrid.fromXContent(p, (String) c));
+		map.put(RangeAggregationBuilder.NAME, (p, c) -> ParsedRange.fromXContent(p, (String) c));
+		map.put(DateRangeAggregationBuilder.NAME, (p, c) -> ParsedDateRange.fromXContent(p, (String) c));
+		map.put(GeoDistanceAggregationBuilder.NAME, (p, c) -> ParsedGeoDistance.fromXContent(p, (String) c));
+		map.put(FiltersAggregationBuilder.NAME, (p, c) -> ParsedFilters.fromXContent(p, (String) c));
+		map.put(AdjacencyMatrixAggregationBuilder.NAME, (p, c) -> ParsedAdjacencyMatrix.fromXContent(p, (String) c));
+		map.put(SignificantLongTerms.NAME, (p, c) -> ParsedSignificantLongTerms.fromXContent(p, (String) c));
+		map.put(SignificantStringTerms.NAME, (p, c) -> ParsedSignificantStringTerms.fromXContent(p, (String) c));
+		map.put(ScriptedMetricAggregationBuilder.NAME, (p, c) -> ParsedScriptedMetric.fromXContent(p, (String) c));
+		map.put(IpRangeAggregationBuilder.NAME, (p, c) -> ParsedBinaryRange.fromXContent(p, (String) c));
+		map.put(TopHitsAggregationBuilder.NAME, (p, c) -> ParsedTopHits.fromXContent(p, (String) c));
+		map.put(CompositeAggregationBuilder.NAME, (p, c) -> ParsedComposite.fromXContent(p, (String) c));
+		List<NamedXContentRegistry.Entry> entries = map.entrySet().stream().map(
+				entry -> new NamedXContentRegistry.Entry(Aggregation.class, new ParseField(entry.getKey()), entry.getValue()))
+				.collect(Collectors.toList());
+		entries.add(
+				new NamedXContentRegistry.Entry(Suggest.Suggestion.class, new ParseField(TermSuggestionBuilder.SUGGESTION_NAME),
+						(parser, context) -> TermSuggestion.fromXContent(parser, (String) context)));
+		entries.add(new NamedXContentRegistry.Entry(Suggest.Suggestion.class,
+				new ParseField(PhraseSuggestionBuilder.SUGGESTION_NAME),
+				(parser, context) -> PhraseSuggestion.fromXContent(parser, (String) context)));
+		entries.add(new NamedXContentRegistry.Entry(Suggest.Suggestion.class,
+				new ParseField(CompletionSuggestionBuilder.SUGGESTION_NAME),
+				(parser, context) -> CompletionSuggestion.fromXContent(parser, (String) context)));
+		return entries;
+	}
+}

src/main/java/org/springframework/data/elasticsearch/client/util/ScrollState.java

@@ -0,0 +1,72 @@
+/*
+ * Copyright 2020 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.springframework.data.elasticsearch.client.util;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.elasticsearch.action.search.SearchScrollRequest;
+import org.elasticsearch.search.Scroll;
+import org.springframework.lang.Nullable;
+import org.springframework.util.StringUtils;
+
+/**
+ * Mutable state object holding scrollId to be used for {@link SearchScrollRequest#scroll(Scroll)}
+ *
+ * @author Christoph Strobl
+ * @author Peter-Josef Meisch
+ * @since 3.2
+ */
+public class ScrollState {
+
+	private final Object lock = new Object();
+
+	private final Set<String> pastIds = new LinkedHashSet<>();
+	@Nullable private String scrollId;
+
+	public ScrollState() {}
+
+	public ScrollState(String scrollId) {
+		updateScrollId(scrollId);
+	}
+
+	@Nullable
+	public String getScrollId() {
+		return scrollId;
+	}
+
+	public List<String> getScrollIds() {
+
+		synchronized (lock) {
+			return Collections.unmodifiableList(new ArrayList<>(pastIds));
+		}
+	}
+
+	public void updateScrollId(String scrollId) {
+
+		if (StringUtils.hasText(scrollId)) {
+
+			synchronized (lock) {
+
+				this.scrollId = scrollId;
+				pastIds.add(scrollId);
+			}
+		}
+	}
+}

src/main/java/org/springframework/data/elasticsearch/config/ElasticsearchAuditingBeanDefinitionParser.java

@@ -0,0 +1,109 @@
+/*
+ * Copyright 2020 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.springframework.data.elasticsearch.config;
+
+import static org.springframework.data.config.ParsingUtils.*;
+
+import org.springframework.beans.factory.support.AbstractBeanDefinition;
+import org.springframework.beans.factory.support.BeanDefinitionBuilder;
+import org.springframework.beans.factory.support.BeanDefinitionRegistry;
+import org.springframework.beans.factory.support.RootBeanDefinition;
+import org.springframework.beans.factory.xml.AbstractSingleBeanDefinitionParser;
+import org.springframework.beans.factory.xml.BeanDefinitionParser;
+import org.springframework.beans.factory.xml.ParserContext;
+import org.springframework.data.auditing.config.IsNewAwareAuditingHandlerBeanDefinitionParser;
+import org.springframework.data.elasticsearch.core.event.AuditingEntityCallback;
+import org.springframework.data.elasticsearch.core.event.ReactiveAuditingEntityCallback;
+import org.springframework.data.elasticsearch.core.mapping.SimpleElasticsearchMappingContext;
+import org.springframework.data.repository.util.ReactiveWrappers;
+import org.springframework.lang.Nullable;
+import org.springframework.util.StringUtils;
+import org.w3c.dom.Element;
+
+/**
+ * {@link BeanDefinitionParser} to register a {@link AuditingEntityCallback} to transparently set auditing information
+ * on an entity.
+ *
+ * @author Peter-Josef Meisch
+ */
+public class ElasticsearchAuditingBeanDefinitionParser extends AbstractSingleBeanDefinitionParser {
+
+	private static String MAPPING_CONTEXT_BEAN_NAME = "simpleElasticsearchMappingContext";
+
+	/*
+	 * (non-Javadoc)
+	 * @see org.springframework.beans.factory.xml.AbstractSingleBeanDefinitionParser#getBeanClass(org.w3c.dom.Element)
+	 */
+	@Override
+	protected Class<?> getBeanClass(Element element) {
+		return AuditingEntityCallback.class;
+	}
+
+	/*
+	 * (non-Javadoc)
+	 * @see org.springframework.beans.factory.xml.AbstractBeanDefinitionParser#shouldGenerateId()
+	 */
+	@Override
+	protected boolean shouldGenerateId() {
+		return true;
+	}
+
+	/*
+	 * (non-Javadoc)
+	 * @see org.springframework.beans.factory.xml.AbstractSingleBeanDefinitionParser#doParse(org.w3c.dom.Element, org.springframework.beans.factory.xml.ParserContext, org.springframework.beans.factory.support.BeanDefinitionBuilder)
+	 */
+	@Override
+	protected void doParse(Element element, ParserContext parserContext, BeanDefinitionBuilder builder) {
+
+		String mappingContextRef = element.getAttribute("mapping-context-ref");
+
+		if (!StringUtils.hasText(mappingContextRef)) {
+
+			BeanDefinitionRegistry registry = parserContext.getRegistry();
+
+			if (!registry.containsBeanDefinition(MAPPING_CONTEXT_BEAN_NAME)) {
+				registry.registerBeanDefinition(MAPPING_CONTEXT_BEAN_NAME,
+						new RootBeanDefinition(SimpleElasticsearchMappingContext.class));
+			}
+
+			mappingContextRef = MAPPING_CONTEXT_BEAN_NAME;
+		}
+
+		IsNewAwareAuditingHandlerBeanDefinitionParser parser = new IsNewAwareAuditingHandlerBeanDefinitionParser(
+				mappingContextRef);
+		parser.parse(element, parserContext);
+
+		AbstractBeanDefinition isNewAwareAuditingHandler = getObjectFactoryBeanDefinition(parser.getResolvedBeanName(),
+				parserContext.extractSource(element));
+		builder.addConstructorArgValue(isNewAwareAuditingHandler);
+
+		if (ReactiveWrappers.isAvailable(ReactiveWrappers.ReactiveLibrary.PROJECT_REACTOR)) {
+			registerReactiveAuditingEntityCallback(parserContext.getRegistry(), isNewAwareAuditingHandler,
+					parserContext.extractSource(element));
+		}
+	}
+
+	private void registerReactiveAuditingEntityCallback(BeanDefinitionRegistry registry,
+			AbstractBeanDefinition isNewAwareAuditingHandler, @Nullable Object source) {
+
+		BeanDefinitionBuilder builder = BeanDefinitionBuilder.rootBeanDefinition(ReactiveAuditingEntityCallback.class);
+
+		builder.addConstructorArgValue(isNewAwareAuditingHandler);
+		builder.getRawBeanDefinition().setSource(source);
+
+		registry.registerBeanDefinition(ReactiveAuditingEntityCallback.class.getName(), builder.getBeanDefinition());
+	}
+}

src/main/java/org/springframework/data/elasticsearch/config/ElasticsearchAuditingRegistrar.java

@@ -0,0 +1,181 @@
+/*
+ * Copyright 2020 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.springframework.data.elasticsearch.config;
+
+import java.lang.annotation.Annotation;
+
+import org.springframework.beans.factory.FactoryBean;
+import org.springframework.beans.factory.config.BeanDefinition;
+import org.springframework.beans.factory.support.AbstractBeanDefinition;
+import org.springframework.beans.factory.support.BeanDefinitionBuilder;
+import org.springframework.beans.factory.support.BeanDefinitionRegistry;
+import org.springframework.context.annotation.ImportBeanDefinitionRegistrar;
+import org.springframework.core.type.AnnotationMetadata;
+import org.springframework.data.auditing.IsNewAwareAuditingHandler;
+import org.springframework.data.auditing.config.AuditingBeanDefinitionRegistrarSupport;
+import org.springframework.data.auditing.config.AuditingConfiguration;
+import org.springframework.data.config.ParsingUtils;
+import org.springframework.data.elasticsearch.core.convert.MappingElasticsearchConverter;
+import org.springframework.data.elasticsearch.core.event.AuditingEntityCallback;
+import org.springframework.data.elasticsearch.core.event.ReactiveAuditingEntityCallback;
+import org.springframework.data.elasticsearch.core.mapping.ElasticsearchPersistentEntity;
+import org.springframework.data.elasticsearch.core.mapping.ElasticsearchPersistentProperty;
+import org.springframework.data.mapping.context.MappingContext;
+import org.springframework.data.repository.util.ReactiveWrappers;
+import org.springframework.util.Assert;
+
+/**
+ * {@link ImportBeanDefinitionRegistrar} to enable {@link EnableElasticsearchAuditing} annotation.
+ *
+ * @author Peter-Josef Meisch
+ * @since 4.0
+ */
+class ElasticsearchAuditingRegistrar extends AuditingBeanDefinitionRegistrarSupport {
+
+	/*
+	 * (non-Javadoc)
+	 * @see org.springframework.data.auditing.config.AuditingBeanDefinitionRegistrarSupport#getAnnotation()
+	 */
+	@Override
+	protected Class<? extends Annotation> getAnnotation() {
+		return EnableElasticsearchAuditing.class;
+	}
+
+	/*
+	 * (non-Javadoc)
+	 * @see org.springframework.data.auditing.config.AuditingBeanDefinitionRegistrarSupport#getAuditingHandlerBeanName()
+	 */
+	@Override
+	protected String getAuditingHandlerBeanName() {
+		return "elasticsearchAuditingHandler";
+	}
+
+	/*
+	 * (non-Javadoc)
+	 * @see org.springframework.data.auditing.config.AuditingBeanDefinitionRegistrarSupport#registerBeanDefinitions(org.springframework.core.type.AnnotationMetadata, org.springframework.beans.factory.support.BeanDefinitionRegistry)
+	 */
+	@Override
+	public void registerBeanDefinitions(AnnotationMetadata annotationMetadata, BeanDefinitionRegistry registry) {
+
+		Assert.notNull(annotationMetadata, "AnnotationMetadata must not be null!");
+		Assert.notNull(registry, "BeanDefinitionRegistry must not be null!");
+
+		super.registerBeanDefinitions(annotationMetadata, registry);
+	}
+
+	/*
+	 * (non-Javadoc)
+	 * @see org.springframework.data.auditing.config.AuditingBeanDefinitionRegistrarSupport#getAuditHandlerBeanDefinitionBuilder(org.springframework.data.auditing.config.AuditingConfiguration)
+	 */
+	@Override
+	protected BeanDefinitionBuilder getAuditHandlerBeanDefinitionBuilder(AuditingConfiguration configuration) {
+
+		Assert.notNull(configuration, "AuditingConfiguration must not be null!");
+
+		BeanDefinitionBuilder builder = BeanDefinitionBuilder.rootBeanDefinition(IsNewAwareAuditingHandler.class);
+
+		BeanDefinitionBuilder definition = BeanDefinitionBuilder
+				.genericBeanDefinition(ElasticsearchMappingContextLookup.class);
+		definition.setAutowireMode(AbstractBeanDefinition.AUTOWIRE_CONSTRUCTOR);
+
+		builder.addConstructorArgValue(definition.getBeanDefinition());
+		return configureDefaultAuditHandlerAttributes(configuration, builder);
+	}
+
+	/*
+	 * (non-Javadoc)
+	 * @see org.springframework.data.auditing.config.AuditingBeanDefinitionRegistrarSupport#registerAuditListener(org.springframework.beans.factory.config.BeanDefinition, org.springframework.beans.factory.support.BeanDefinitionRegistry)
+	 */
+	@Override
+	protected void registerAuditListenerBeanDefinition(BeanDefinition auditingHandlerDefinition,
+			BeanDefinitionRegistry registry) {
+
+		Assert.notNull(auditingHandlerDefinition, "BeanDefinition must not be null!");
+		Assert.notNull(registry, "BeanDefinitionRegistry must not be null!");
+
+		BeanDefinitionBuilder listenerBeanDefinitionBuilder = BeanDefinitionBuilder
+				.rootBeanDefinition(AuditingEntityCallback.class);
+		listenerBeanDefinitionBuilder
+				.addConstructorArgValue(ParsingUtils.getObjectFactoryBeanDefinition(getAuditingHandlerBeanName(), registry));
+
+		registerInfrastructureBeanWithId(listenerBeanDefinitionBuilder.getBeanDefinition(),
+				AuditingEntityCallback.class.getName(), registry);
+
+		if (ReactiveWrappers.isAvailable(ReactiveWrappers.ReactiveLibrary.PROJECT_REACTOR)) {
+			registerReactiveAuditingEntityCallback(registry, auditingHandlerDefinition.getSource());
+		}
+	}
+
+	private void registerReactiveAuditingEntityCallback(BeanDefinitionRegistry registry, Object source) {
+
+		BeanDefinitionBuilder builder = BeanDefinitionBuilder.rootBeanDefinition(ReactiveAuditingEntityCallback.class);
+
+		builder.addConstructorArgValue(ParsingUtils.getObjectFactoryBeanDefinition(getAuditingHandlerBeanName(), registry));
+		builder.getRawBeanDefinition().setSource(source);
+
+		registerInfrastructureBeanWithId(builder.getBeanDefinition(), ReactiveAuditingEntityCallback.class.getName(),
+				registry);
+	}
+
+	/**
+	 * Simple helper to be able to wire the {@link MappingContext} from a {@link MappingElasticsearchConverter} bean
+	 * available in the application context.
+	 *
+	 * @author Oliver Gierke
+	 */
+	static class ElasticsearchMappingContextLookup implements
+			FactoryBean<MappingContext<? extends ElasticsearchPersistentEntity<?>, ElasticsearchPersistentProperty>> {
+
+		private final MappingElasticsearchConverter converter;
+
+		/**
+		 * Creates a new {@link ElasticsearchMappingContextLookup} for the given {@link MappingElasticsearchConverter}.
+		 *
+		 * @param converter must not be {@literal null}.
+		 */
+		public ElasticsearchMappingContextLookup(MappingElasticsearchConverter converter) {
+			this.converter = converter;
+		}
+
+		/*
+		 * (non-Javadoc)
+		 * @see org.springframework.beans.factory.FactoryBean#getObject()
+		 */
+		@Override
+		public MappingContext<? extends ElasticsearchPersistentEntity<?>, ElasticsearchPersistentProperty> getObject()
+				throws Exception {
+			return converter.getMappingContext();
+		}
+
+		/*
+		 * (non-Javadoc)
+		 * @see org.springframework.beans.factory.FactoryBean#getObjectType()
+		 */
+		@Override
+		public Class<?> getObjectType() {
+			return MappingContext.class;
+		}
+
+		/*
+		 * (non-Javadoc)
+		 * @see org.springframework.beans.factory.FactoryBean#isSingleton()
+		 */
+		@Override
+		public boolean isSingleton() {
+			return true;
+		}
+	}
+}

src/main/java/org/springframework/data/elasticsearch/config/ElasticsearchConfigurationSupport.java

@@ -15,8 +15,6 @@
  */
 package org.springframework.data.elasticsearch.config;
 
-import lombok.SneakyThrows;
-
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashSet;
@@ -62,7 +60,6 @@ public class ElasticsearchConfigurationSupport {
 	 * @return never {@literal null}.
 	 */
 	@Bean
-	@SneakyThrows
 	public SimpleElasticsearchMappingContext elasticsearchMappingContext() {
 
 		SimpleElasticsearchMappingContext mappingContext = new SimpleElasticsearchMappingContext();
@@ -103,9 +100,8 @@ public class ElasticsearchConfigurationSupport {
 	 *
 	 * @see #getMappingBasePackages()
 	 * @return never {@literal null}.
-	 * @throws ClassNotFoundException
 	 */
-	protected Set<Class<?>> getInitialEntitySet() throws ClassNotFoundException {
+	protected Set<Class<?>> getInitialEntitySet() {
 
 		Set<Class<?>> initialEntitySet = new HashSet<>();
 
@@ -122,9 +118,8 @@ public class ElasticsearchConfigurationSupport {
 	 *
 	 * @param basePackage must not be {@literal null}.
 	 * @return never {@literal null}.
-	 * @throws ClassNotFoundException
 	 */
-	protected Set<Class<?>> scanForEntities(String basePackage) throws ClassNotFoundException {
+	protected Set<Class<?>> scanForEntities(String basePackage) {
 
 		if (!StringUtils.hasText(basePackage)) {
 			return Collections.emptySet();
@@ -132,17 +127,20 @@ public class ElasticsearchConfigurationSupport {
 
 		Set<Class<?>> initialEntitySet = new HashSet<>();
 
-		if (StringUtils.hasText(basePackage)) {
+		ClassPathScanningCandidateComponentProvider componentProvider = new ClassPathScanningCandidateComponentProvider(
+				false);
+		componentProvider.addIncludeFilter(new AnnotationTypeFilter(Document.class));
+		componentProvider.addIncludeFilter(new AnnotationTypeFilter(Persistent.class));
 
-			ClassPathScanningCandidateComponentProvider componentProvider = new ClassPathScanningCandidateComponentProvider(
-					false);
-			componentProvider.addIncludeFilter(new AnnotationTypeFilter(Document.class));
-			componentProvider.addIncludeFilter(new AnnotationTypeFilter(Persistent.class));
+		for (BeanDefinition candidate : componentProvider.findCandidateComponents(basePackage)) {
 
-			for (BeanDefinition candidate : componentProvider.findCandidateComponents(basePackage)) {
+			String beanClassName = candidate.getBeanClassName();
 
-				initialEntitySet.add(ClassUtils.forName(candidate.getBeanClassName(),
-						AbstractReactiveElasticsearchConfiguration.class.getClassLoader()));
+			if (beanClassName != null) {
+				try {
+					initialEntitySet.add(
+							ClassUtils.forName(beanClassName, AbstractReactiveElasticsearchConfiguration.class.getClassLoader()));
+				} catch (ClassNotFoundException | LinkageError ignored) {}
 			}
 		}
 

src/main/java/org/springframework/data/elasticsearch/config/EnableElasticsearchAuditing.java

@@ -0,0 +1,70 @@
+/*
+ * Copyright 2020 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.springframework.data.elasticsearch.config;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Inherited;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+import org.springframework.context.annotation.Import;
+import org.springframework.data.auditing.DateTimeProvider;
+import org.springframework.data.domain.AuditorAware;
+
+/**
+ * Annotation to enable auditing in Elasticsearch via annotation configuration.
+ *
+ * @author Peter-Josef Meisch
+ * @since 4.0
+ */
+@Inherited
+@Documented
+@Target(ElementType.TYPE)
+@Retention(RetentionPolicy.RUNTIME)
+@Import(ElasticsearchAuditingRegistrar.class)
+public @interface EnableElasticsearchAuditing {
+
+	/**
+	 * Configures the {@link AuditorAware} bean to be used to lookup the current principal.
+	 *
+	 * @return
+	 */
+	String auditorAwareRef() default "";
+
+	/**
+	 * Configures whether the creation and modification dates are set. Defaults to {@literal true}.
+	 *
+	 * @return
+	 */
+	boolean setDates() default true;
+
+	/**
+	 * Configures whether the entity shall be marked as modified on creation. Defaults to {@literal true}.
+	 *
+	 * @return
+	 */
+	boolean modifyOnCreate() default true;
+
+	/**
+	 * Configures a {@link DateTimeProvider} bean name that allows customizing the {@link org.joda.time.DateTime} to be
+	 * used for setting creation and modification dates.
+	 *
+	 * @return
+	 */
+	String dateTimeProviderRef() default "";
+}

src/main/java/org/springframework/data/elasticsearch/core/AbstractDefaultIndexOperations.java

@@ -55,7 +55,7 @@ abstract class AbstractDefaultIndexOperations implements IndexOperations {
 	protected final RequestFactory requestFactory;
 
 	@Nullable protected final Class<?> boundClass;
-	protected final IndexCoordinates boundIndex;
+	private final IndexCoordinates boundIndex;
 
 	public AbstractDefaultIndexOperations(ElasticsearchConverter elasticsearchConverter, Class<?> boundClass) {
 		this.elasticsearchConverter = elasticsearchConverter;
@@ -87,7 +87,7 @@ abstract class AbstractDefaultIndexOperations implements IndexOperations {
 
 		if (boundClass != null) {
 			Class<?> clazz = boundClass;
-			String indexName = boundIndex.getIndexName();
+			String indexName = getIndexCoordinates().getIndexName();
 
 			if (clazz.isAnnotationPresent(Setting.class)) {
 				String settingPath = clazz.getAnnotation(Setting.class).settingPath();
@@ -104,40 +104,40 @@ abstract class AbstractDefaultIndexOperations implements IndexOperations {
 			}
 			return doCreate(indexName, getDefaultSettings(getRequiredPersistentEntity(clazz)));
 		}
-		return doCreate(boundIndex.getIndexName(), null);
+		return doCreate(getIndexCoordinates().getIndexName(), null);
 	}
 
 	@Override
 	public boolean create(Document settings) {
-		return doCreate(boundIndex.getIndexName(), settings);
+		return doCreate(getIndexCoordinates().getIndexName(), settings);
 	}
 
 	protected abstract boolean doCreate(String indexName, @Nullable Document settings);
 
 	@Override
 	public boolean delete() {
-		return doDelete(boundIndex.getIndexName());
+		return doDelete(getIndexCoordinates().getIndexName());
 	}
 
 	protected abstract boolean doDelete(String indexName);
 
 	@Override
 	public boolean exists() {
-		return doExists(boundIndex.getIndexName());
+		return doExists(getIndexCoordinates().getIndexName());
 	}
 
 	protected abstract boolean doExists(String indexName);
 
 	@Override
 	public boolean putMapping(Document mapping) {
-		return doPutMapping(boundIndex, mapping);
+		return doPutMapping(getIndexCoordinates(), mapping);
 	}
 
 	protected abstract boolean doPutMapping(IndexCoordinates index, Document mapping);
 
 	@Override
 	public Map<String, Object> getMapping() {
-		return doGetMapping(boundIndex);
+		return doGetMapping(getIndexCoordinates());
 	}
 
 	abstract protected Map<String, Object> doGetMapping(IndexCoordinates index);
@@ -149,35 +149,35 @@ abstract class AbstractDefaultIndexOperations implements IndexOperations {
 
 	@Override
 	public Map<String, Object> getSettings(boolean includeDefaults) {
-		return doGetSettings(boundIndex.getIndexName(), includeDefaults);
+		return doGetSettings(getIndexCoordinates().getIndexName(), includeDefaults);
 	}
 
 	protected abstract Map<String, Object> doGetSettings(String indexName, boolean includeDefaults);
 
 	@Override
 	public void refresh() {
-		doRefresh(boundIndex);
+		doRefresh(getIndexCoordinates());
 	}
 
 	protected abstract void doRefresh(IndexCoordinates indexCoordinates);
 
 	@Override
 	public boolean addAlias(AliasQuery query) {
-		return doAddAlias(query, boundIndex);
+		return doAddAlias(query, getIndexCoordinates());
 	}
 
 	protected abstract boolean doAddAlias(AliasQuery query, IndexCoordinates index);
 
 	@Override
 	public List<AliasMetaData> queryForAlias() {
-		return doQueryForAlias(boundIndex.getIndexName());
+		return doQueryForAlias(getIndexCoordinates().getIndexName());
 	}
 
 	protected abstract List<AliasMetaData> doQueryForAlias(String indexName);
 
 	@Override
 	public boolean removeAlias(AliasQuery query) {
-		return doRemoveAlias(query, boundIndex);
+		return doRemoveAlias(query, getIndexCoordinates());
 	}
 
 	protected abstract boolean doRemoveAlias(AliasQuery query, IndexCoordinates index);
@@ -238,6 +238,16 @@ abstract class AbstractDefaultIndexOperations implements IndexOperations {
 		return elasticsearchConverter.getMappingContext().getRequiredPersistentEntity(clazz);
 	}
 
+	/**
+	 * get the current {@link IndexCoordinates}. These may change over time when the entity class has a SpEL constructed
+	 * index name. When this IndexOperations is not bound to a class, the bound IndexCoordinates are returned.
+	 *
+	 * @return IndexCoordinates
+	 */
+	protected IndexCoordinates getIndexCoordinates() {
+		return (boundClass != null) ? getIndexCoordinatesFor(boundClass) : boundIndex;
+	}
+
 	public IndexCoordinates getIndexCoordinatesFor(Class<?> clazz) {
 		return getRequiredPersistentEntity(clazz).getIndexCoordinates();
 	}

src/main/java/org/springframework/data/elasticsearch/core/AbstractElasticsearchTemplate.java

@@ -1,12 +1,27 @@
+/*
+ * Copyright 2019-2020 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 package org.springframework.data.elasticsearch.core;
 
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
-import java.util.Optional;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
@@ -14,15 +29,21 @@ import org.elasticsearch.action.bulk.BulkItemResponse;
 import org.elasticsearch.action.bulk.BulkResponse;
 import org.elasticsearch.action.search.MultiSearchRequest;
 import org.elasticsearch.action.search.MultiSearchResponse;
+import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.query.MoreLikeThisQueryBuilder;
 import org.springframework.beans.BeansException;
 import org.springframework.context.ApplicationContext;
 import org.springframework.context.ApplicationContextAware;
-import org.springframework.data.elasticsearch.ElasticsearchException;
+import org.springframework.data.convert.EntityReader;
+import org.springframework.data.elasticsearch.BulkFailureException;
 import org.springframework.data.elasticsearch.core.convert.ElasticsearchConverter;
 import org.springframework.data.elasticsearch.core.convert.MappingElasticsearchConverter;
+import org.springframework.data.elasticsearch.core.document.Document;
 import org.springframework.data.elasticsearch.core.document.SearchDocumentResponse;
+import org.springframework.data.elasticsearch.core.event.AfterConvertCallback;
+import org.springframework.data.elasticsearch.core.event.AfterSaveCallback;
+import org.springframework.data.elasticsearch.core.event.BeforeConvertCallback;
 import org.springframework.data.elasticsearch.core.mapping.ElasticsearchPersistentEntity;
 import org.springframework.data.elasticsearch.core.mapping.ElasticsearchPersistentProperty;
 import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates;
@@ -33,8 +54,12 @@ import org.springframework.data.elasticsearch.core.query.IndexQueryBuilder;
 import org.springframework.data.elasticsearch.core.query.MoreLikeThisQuery;
 import org.springframework.data.elasticsearch.core.query.NativeSearchQueryBuilder;
 import org.springframework.data.elasticsearch.core.query.Query;
+import org.springframework.data.elasticsearch.core.query.SeqNoPrimaryTerm;
+import org.springframework.data.elasticsearch.support.VersionInfo;
+import org.springframework.data.mapping.callback.EntityCallbacks;
 import org.springframework.data.util.CloseableIterator;
 import org.springframework.data.util.Streamable;
+import org.springframework.lang.NonNull;
 import org.springframework.lang.Nullable;
 import org.springframework.util.Assert;
 
@@ -43,12 +68,15 @@ import org.springframework.util.Assert;
  *
  * @author Sascha Woo
  * @author Peter-Josef Meisch
+ * @author Roman Puchkovskiy
  */
 public abstract class AbstractElasticsearchTemplate implements ElasticsearchOperations, ApplicationContextAware {
 
 	protected @Nullable ElasticsearchConverter elasticsearchConverter;
 	protected @Nullable RequestFactory requestFactory;
 
+	private @Nullable EntityCallbacks entityCallbacks;
+
 	// region Initialization
 	protected void initialize(ElasticsearchConverter elasticsearchConverter) {
 
@@ -56,6 +84,8 @@ public abstract class AbstractElasticsearchTemplate implements ElasticsearchOper
 
 		this.elasticsearchConverter = elasticsearchConverter;
 		requestFactory = new RequestFactory(elasticsearchConverter);
+
+		VersionInfo.logVersions(getClusterVersion());
 	}
 
 	protected ElasticsearchConverter createElasticsearchConverter() {
@@ -66,16 +96,36 @@ public abstract class AbstractElasticsearchTemplate implements ElasticsearchOper
 	}
 
 	@Override
-	public void setApplicationContext(ApplicationContext context) throws BeansException {
+	public void setApplicationContext(ApplicationContext applicationContext) throws BeansException {
+
+		if (entityCallbacks == null) {
+			setEntityCallbacks(EntityCallbacks.create(applicationContext));
+		}
 
 		if (elasticsearchConverter instanceof ApplicationContextAware) {
-			((ApplicationContextAware) elasticsearchConverter).setApplicationContext(context);
+			((ApplicationContextAware) elasticsearchConverter).setApplicationContext(applicationContext);
 		}
 	}
+
+	/**
+	 * Set the {@link EntityCallbacks} instance to use when invoking {@link EntityCallbacks callbacks} like the
+	 * {@link org.springframework.data.elasticsearch.core.event.BeforeConvertCallback}.
+	 * <p />
+	 * Overrides potentially existing {@link EntityCallbacks}.
+	 *
+	 * @param entityCallbacks must not be {@literal null}.
+	 * @throws IllegalArgumentException if the given instance is {@literal null}.
+	 * @since 4.0
+	 */
+	public void setEntityCallbacks(EntityCallbacks entityCallbacks) {
+
+		Assert.notNull(entityCallbacks, "entityCallbacks must not be null");
+
+		this.entityCallbacks = entityCallbacks;
+	}
 	// endregion
 
 	// region DocumentOperations
-
 	@Override
 	public <T> T save(T entity) {
 
@@ -90,8 +140,13 @@ public abstract class AbstractElasticsearchTemplate implements ElasticsearchOper
 		Assert.notNull(entity, "entity must not be null");
 		Assert.notNull(index, "index must not be null");
 
-		index(getIndexQuery(entity), index);
-		return entity;
+		IndexQuery query = getIndexQuery(entity);
+		index(query, index);
+
+		// suppressing because it's either entity itself or something of a correct type returned by an entity callback
+		@SuppressWarnings("unchecked")
+		T castResult = (T) query.getObject();
+		return castResult;
 	}
 
 	@Override
@@ -124,7 +179,7 @@ public abstract class AbstractElasticsearchTemplate implements ElasticsearchOper
 			});
 		}
 
-		return entities;
+		return indexQueries.stream().map(IndexQuery::getObject).map(entity -> (T) entity).collect(Collectors.toList());
 	}
 
 	@Override
@@ -144,6 +199,12 @@ public abstract class AbstractElasticsearchTemplate implements ElasticsearchOper
 		return get(query.getId(), clazz, index);
 	}
 
+	@Override
+	@Nullable
+	public <T> T queryForObject(GetQuery query, Class<T> clazz) {
+		return get(query.getId(), clazz, getIndexCoordinatesFor(clazz));
+	}
+
 	@Override
 	public boolean exists(String id, Class<?> clazz) {
 		return exists(id, getIndexCoordinatesFor(clazz));
@@ -184,20 +245,27 @@ public abstract class AbstractElasticsearchTemplate implements ElasticsearchOper
 
 	@Override
 	public <T> CloseableIterator<T> stream(Query query, Class<T> clazz, IndexCoordinates index) {
-		long scrollTimeInMillis = TimeValue.timeValueMinutes(1).millis();
 		return (CloseableIterator<T>) SearchHitSupport.unwrapSearchHits(searchForStream(query, clazz, index));
 	}
 
 	@Override
-	public <T> CloseableIterator<SearchHit<T>> searchForStream(Query query, Class<T> clazz) {
+	public <T> SearchHitsIterator<T> searchForStream(Query query, Class<T> clazz) {
 		return searchForStream(query, clazz, getIndexCoordinatesFor(clazz));
 	}
 
 	@Override
-	public <T> CloseableIterator<SearchHit<T>> searchForStream(Query query, Class<T> clazz, IndexCoordinates index) {
+	public <T> SearchHitsIterator<T> searchForStream(Query query, Class<T> clazz, IndexCoordinates index) {
+
 		long scrollTimeInMillis = TimeValue.timeValueMinutes(1).millis();
-		return StreamQueries.streamResults(searchScrollStart(scrollTimeInMillis, query, clazz, index),
-				scrollId -> searchScrollContinue(scrollId, scrollTimeInMillis, clazz), this::searchScrollClear);
+
+		// noinspection ConstantConditions
+		int maxCount = query.isLimiting() ? query.getMaxResults() : 0;
+
+		return StreamQueries.streamResults( //
+				maxCount, //
+				searchScrollStart(scrollTimeInMillis, query, clazz, index), //
+				scrollId -> searchScrollContinue(scrollId, scrollTimeInMillis, clazz, index), //
+				this::searchScrollClear);
 	}
 
 	@Override
@@ -223,10 +291,11 @@ public abstract class AbstractElasticsearchTemplate implements ElasticsearchOper
 
 		MultiSearchResponse.Item[] items = getMultiSearchResult(request);
 
+		SearchDocumentResponseCallback<SearchHits<T>> callback = new ReadSearchDocumentResponseCallback<>(clazz, index);
 		List<SearchHits<T>> res = new ArrayList<>(queries.size());
 		int c = 0;
 		for (Query query : queries) {
-			res.add(elasticsearchConverter.read(clazz, SearchDocumentResponse.from(items[c++].getResponse())));
+			res.add(callback.doWith(SearchDocumentResponse.from(items[c++].getResponse())));
 		}
 		return res;
 	}
@@ -246,7 +315,13 @@ public abstract class AbstractElasticsearchTemplate implements ElasticsearchOper
 		int c = 0;
 		Iterator<Class<?>> it1 = classes.iterator();
 		for (Query query : queries) {
-			res.add(elasticsearchConverter.read(it1.next(), SearchDocumentResponse.from(items[c++].getResponse())));
+			Class entityClass = it1.next();
+
+			SearchDocumentResponseCallback<SearchHits<?>> callback = new ReadSearchDocumentResponseCallback<>(entityClass,
+					index);
+
+			SearchResponse response = items[c++].getResponse();
+			res.add(callback.doWith(SearchDocumentResponse.from(response)));
 		}
 		return res;
 	}
@@ -259,19 +334,26 @@ public abstract class AbstractElasticsearchTemplate implements ElasticsearchOper
 	/*
 	 * internal use only, not for public API
 	 */
-	abstract protected <T> ScrolledPage<SearchHit<T>> searchScrollStart(long scrollTimeInMillis, Query query,
+	abstract protected <T> SearchScrollHits<T> searchScrollStart(long scrollTimeInMillis, Query query, Class<T> clazz,
+			IndexCoordinates index);
+
+	/*
+	 * internal use only, not for public API
+	 */
+	abstract protected <T> SearchScrollHits<T> searchScrollContinue(@Nullable String scrollId, long scrollTimeInMillis,
 			Class<T> clazz, IndexCoordinates index);
 
 	/*
 	 * internal use only, not for public API
 	 */
-	abstract protected <T> ScrolledPage<SearchHit<T>> searchScrollContinue(@Nullable String scrollId,
-			long scrollTimeInMillis, Class<T> clazz);
+	protected void searchScrollClear(String scrollId) {
+		searchScrollClear(Collections.singletonList(scrollId));
+	}
 
 	/*
 	 * internal use only, not for public API
 	 */
-	abstract protected void searchScrollClear(String scrollId);
+	abstract protected void searchScrollClear(List<String> scrollIds);
 
 	abstract protected MultiSearchResponse.Item[] getMultiSearchResult(MultiSearchRequest request);
 	// endregion
@@ -323,7 +405,7 @@ public abstract class AbstractElasticsearchTemplate implements ElasticsearchOper
 				if (item.isFailed())
 					failedDocuments.put(item.getId(), item.getFailureMessage());
 			}
-			throw new ElasticsearchException(
+			throw new BulkFailureException(
 					"Bulk operation has failures. Use ElasticsearchException.getFailedDocuments() for detailed messages ["
 							+ failedDocuments + ']',
 					failedDocuments);
@@ -375,6 +457,22 @@ public abstract class AbstractElasticsearchTemplate implements ElasticsearchOper
 		return null;
 	}
 
+	@Nullable
+	private SeqNoPrimaryTerm getEntitySeqNoPrimaryTerm(Object entity) {
+		ElasticsearchPersistentEntity<?> persistentEntity = getRequiredPersistentEntity(entity.getClass());
+		ElasticsearchPersistentProperty property = persistentEntity.getSeqNoPrimaryTermProperty();
+
+		if (property != null) {
+			Object seqNoPrimaryTerm = persistentEntity.getPropertyAccessor(entity).getProperty(property);
+
+			if (seqNoPrimaryTerm != null && SeqNoPrimaryTerm.class.isAssignableFrom(seqNoPrimaryTerm.getClass())) {
+				return (SeqNoPrimaryTerm) seqNoPrimaryTerm;
+			}
+		}
+
+		return null;
+	}
+
 	private <T> IndexQuery getIndexQuery(T entity) {
 		String id = getEntityId(entity);
 
@@ -382,12 +480,173 @@ public abstract class AbstractElasticsearchTemplate implements ElasticsearchOper
 			id = elasticsearchConverter.convertId(id);
 		}
 
-		return new IndexQueryBuilder() //
+		IndexQueryBuilder builder = new IndexQueryBuilder() //
 				.withId(id) //
-				.withVersion(getEntityVersion(entity)) //
-				.withObject(entity) //
-				.build();
+				.withObject(entity);
+		SeqNoPrimaryTerm seqNoPrimaryTerm = getEntitySeqNoPrimaryTerm(entity);
+		if (seqNoPrimaryTerm != null) {
+			builder.withSeqNoPrimaryTerm(seqNoPrimaryTerm);
+		} else {
+			// version cannot be used together with seq_no and primary_term
+			builder.withVersion(getEntityVersion(entity));
+		}
+		return builder.build();
+	}
+
+	/**
+	 * tries to extract the version of the Elasticsearch cluster
+	 *
+	 * @return the version as string if it can be retrieved
+	 */
+	@Nullable
+	abstract protected String getClusterVersion();
+
+	// endregion
+
+	// region Entity callbacks
+	protected <T> T maybeCallbackBeforeConvert(T entity, IndexCoordinates index) {
+
+		if (entityCallbacks != null) {
+			return entityCallbacks.callback(BeforeConvertCallback.class, entity, index);
+		}
+
+		return entity;
+	}
+
+	protected void maybeCallbackBeforeConvertWithQuery(Object query, IndexCoordinates index) {
+
+		if (query instanceof IndexQuery) {
+			IndexQuery indexQuery = (IndexQuery) query;
+			Object queryObject = indexQuery.getObject();
+
+			if (queryObject != null) {
+				queryObject = maybeCallbackBeforeConvert(queryObject, index);
+				indexQuery.setObject(queryObject);
+			}
+		}
+	}
+
+	// this can be called with either a List<IndexQuery> or a List<UpdateQuery>; these query classes
+	// don't have a common base class, therefore the List<?> argument
+	protected void maybeCallbackBeforeConvertWithQueries(List<?> queries, IndexCoordinates index) {
+		queries.forEach(query -> maybeCallbackBeforeConvertWithQuery(query, index));
+	}
+
+	protected <T> T maybeCallbackAfterSave(T entity, IndexCoordinates index) {
+
+		if (entityCallbacks != null) {
+			return entityCallbacks.callback(AfterSaveCallback.class, entity, index);
+		}
+
+		return entity;
+	}
+
+	protected void maybeCallbackAfterSaveWithQuery(Object query, IndexCoordinates index) {
+
+		if (query instanceof IndexQuery) {
+			IndexQuery indexQuery = (IndexQuery) query;
+			Object queryObject = indexQuery.getObject();
+
+			if (queryObject != null) {
+				queryObject = maybeCallbackAfterSave(queryObject, index);
+				indexQuery.setObject(queryObject);
+			}
+		}
+	}
+
+	// this can be called with either a List<IndexQuery> or a List<UpdateQuery>; these query classes
+	// don't have a common base class, therefore the List<?> argument
+	protected void maybeCallbackAfterSaveWithQueries(List<?> queries, IndexCoordinates index) {
+		queries.forEach(query -> maybeCallbackAfterSaveWithQuery(query, index));
+	}
+
+	protected <T> T maybeCallbackAfterConvert(T entity, Document document, IndexCoordinates index) {
+
+		if (entityCallbacks != null) {
+			return entityCallbacks.callback(AfterConvertCallback.class, entity, document, index);
+		}
+
+		return entity;
 	}
 
 	// endregion
+
+	// region Document callbacks
+	protected interface DocumentCallback<T> {
+		@Nullable
+		T doWith(@Nullable Document document);
+	}
+
+	protected class ReadDocumentCallback<T> implements DocumentCallback<T> {
+		private final EntityReader<? super T, Document> reader;
+		private final Class<T> type;
+		private final IndexCoordinates index;
+
+		public ReadDocumentCallback(EntityReader<? super T, Document> reader, Class<T> type, IndexCoordinates index) {
+
+			Assert.notNull(reader, "reader is null");
+			Assert.notNull(type, "type is null");
+
+			this.reader = reader;
+			this.type = type;
+			this.index = index;
+		}
+
+		@Nullable
+		public T doWith(@Nullable Document document) {
+
+			if (document == null) {
+				return null;
+			}
+
+			T entity = reader.read(type, document);
+			return maybeCallbackAfterConvert(entity, document, index);
+		}
+	}
+
+	protected interface SearchDocumentResponseCallback<T> {
+		@NonNull
+		T doWith(@NonNull SearchDocumentResponse response);
+	}
+
+	protected class ReadSearchDocumentResponseCallback<T> implements SearchDocumentResponseCallback<SearchHits<T>> {
+		private final DocumentCallback<T> delegate;
+		private final Class<T> type;
+
+		public ReadSearchDocumentResponseCallback(Class<T> type, IndexCoordinates index) {
+
+			Assert.notNull(type, "type is null");
+
+			this.delegate = new ReadDocumentCallback<>(elasticsearchConverter, type, index);
+			this.type = type;
+		}
+
+		@Override
+		public SearchHits<T> doWith(SearchDocumentResponse response) {
+			List<T> entities = response.getSearchDocuments().stream().map(delegate::doWith).collect(Collectors.toList());
+			return SearchHitMapping.mappingFor(type, elasticsearchConverter.getMappingContext()).mapHits(response, entities);
+		}
+	}
+
+	protected class ReadSearchScrollDocumentResponseCallback<T>
+			implements SearchDocumentResponseCallback<SearchScrollHits<T>> {
+		private final DocumentCallback<T> delegate;
+		private final Class<T> type;
+
+		public ReadSearchScrollDocumentResponseCallback(Class<T> type, IndexCoordinates index) {
+
+			Assert.notNull(type, "type is null");
+
+			this.delegate = new ReadDocumentCallback<>(elasticsearchConverter, type, index);
+			this.type = type;
+		}
+
+		@Override
+		public SearchScrollHits<T> doWith(SearchDocumentResponse response) {
+			List<T> entities = response.getSearchDocuments().stream().map(delegate::doWith).collect(Collectors.toList());
+			return SearchHitMapping.mappingFor(type, elasticsearchConverter.getMappingContext()).mapScrollHits(response,
+					entities);
+		}
+	}
+	// endregion
 }

src/main/java/org/springframework/data/elasticsearch/core/CriteriaQueryProcessor.java

@@ -29,7 +29,9 @@ import org.apache.lucene.queryparser.flexible.core.util.StringUtils;
 import org.apache.lucene.queryparser.flexible.standard.QueryParserUtil;
 import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
+import org.springframework.data.elasticsearch.annotations.FieldType;
 import org.springframework.data.elasticsearch.core.query.Criteria;
+import org.springframework.data.elasticsearch.core.query.Field;
 import org.springframework.lang.Nullable;
 import org.springframework.util.Assert;
 
@@ -118,18 +120,19 @@ class CriteriaQueryProcessor {
 		Iterator<Criteria.CriteriaEntry> it = chainedCriteria.getQueryCriteriaEntries().iterator();
 		boolean singeEntryCriteria = (chainedCriteria.getQueryCriteriaEntries().size() == 1);
 
-		String fieldName = chainedCriteria.getField().getName();
+		Field field = chainedCriteria.getField();
+		String fieldName = field.getName();
 		Assert.notNull(fieldName, "Unknown field");
 		QueryBuilder query = null;
 
 		if (singeEntryCriteria) {
 			Criteria.CriteriaEntry entry = it.next();
-			query = processCriteriaEntry(entry, fieldName);
+			query = processCriteriaEntry(entry, field);
 		} else {
 			query = boolQuery();
 			while (it.hasNext()) {
 				Criteria.CriteriaEntry entry = it.next();
-				((BoolQueryBuilder) query).must(processCriteriaEntry(entry, fieldName));
+				((BoolQueryBuilder) query).must(processCriteriaEntry(entry, field));
 			}
 		}
 
@@ -138,7 +141,11 @@ class CriteriaQueryProcessor {
 	}
 
 	@Nullable
-	private QueryBuilder processCriteriaEntry(Criteria.CriteriaEntry entry, String fieldName) {
+	private QueryBuilder processCriteriaEntry(Criteria.CriteriaEntry entry, Field field) {
+
+		String fieldName = field.getName();
+		boolean isKeywordField = FieldType.Keyword == field.getFieldType();
+
 		OperationKey key = entry.getKey();
 		Object value = entry.getValue();
 
@@ -191,10 +198,24 @@ class CriteriaQueryProcessor {
 				query = fuzzyQuery(fieldName, searchText);
 				break;
 			case IN:
-				query = boolQuery().must(termsQuery(fieldName, toStringList((Iterable<Object>) value)));
+				if (value instanceof Iterable) {
+					Iterable<?> iterable = (Iterable<?>) value;
+					if (isKeywordField) {
+						query = boolQuery().must(termsQuery(fieldName, toStringList(iterable)));
+					} else {
+						query = queryStringQuery(orQueryString(iterable)).field(fieldName);
+					}
+				}
 				break;
 			case NOT_IN:
-				query = boolQuery().mustNot(termsQuery(fieldName, toStringList((Iterable<Object>) value)));
+				if (value instanceof Iterable) {
+					Iterable<?> iterable = (Iterable<?>) value;
+					if (isKeywordField) {
+						query = boolQuery().mustNot(termsQuery(fieldName, toStringList(iterable)));
+					} else {
+						query = queryStringQuery("NOT(" + orQueryString(iterable) + ')').field(fieldName);
+					}
+				}
 				break;
 		}
 		return query;
@@ -208,6 +229,25 @@ class CriteriaQueryProcessor {
 		return list;
 	}
 
+	private static String orQueryString(Iterable<?> iterable) {
+		StringBuilder sb = new StringBuilder();
+
+		for (Object item : iterable) {
+
+			if (item != null) {
+
+				if (sb.length() > 0) {
+					sb.append(' ');
+				}
+				sb.append('"');
+				sb.append(QueryParserUtil.escape(item.toString()));
+				sb.append('"');
+			}
+		}
+
+		return sb.toString();
+	}
+
 	private void addBoost(QueryBuilder query, float boost) {
 		if (Float.isNaN(boost)) {
 			return;

src/main/java/org/springframework/data/elasticsearch/core/DefaultIndexOperations.java

@@ -33,14 +33,12 @@ import org.elasticsearch.client.Request;
 import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.RestClient;
-import org.elasticsearch.client.RestHighLevelClient;
 import org.elasticsearch.client.indices.CreateIndexRequest;
 import org.elasticsearch.client.indices.GetIndexRequest;
 import org.elasticsearch.client.indices.PutMappingRequest;
 import org.elasticsearch.cluster.metadata.AliasMetaData;
-import org.springframework.data.elasticsearch.ElasticsearchException;
+import org.springframework.data.elasticsearch.UncategorizedElasticsearchException;
 import org.springframework.data.elasticsearch.core.client.support.AliasData;
-import org.springframework.data.elasticsearch.core.convert.ElasticsearchConverter;
 import org.springframework.data.elasticsearch.core.document.Document;
 import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates;
 import org.springframework.data.elasticsearch.core.query.AliasQuery;
@@ -60,29 +58,22 @@ import com.fasterxml.jackson.databind.ObjectMapper;
  */
 class DefaultIndexOperations extends AbstractDefaultIndexOperations implements IndexOperations {
 
-	private RestHighLevelClient client;
+	private ElasticsearchRestTemplate restTemplate;
 
-	public DefaultIndexOperations(RestHighLevelClient client, ElasticsearchConverter elasticsearchConverter,
-			Class<?> boundClass) {
-		super(elasticsearchConverter, boundClass);
-		this.client = client;
+	public DefaultIndexOperations(ElasticsearchRestTemplate restTemplate, Class<?> boundClass) {
+		super(restTemplate.getElasticsearchConverter(), boundClass);
+		this.restTemplate = restTemplate;
 	}
 
-	public DefaultIndexOperations(RestHighLevelClient client, ElasticsearchConverter elasticsearchConverter,
-			IndexCoordinates boundIndex) {
-		super(elasticsearchConverter, boundIndex);
-		this.client = client;
+	public DefaultIndexOperations(ElasticsearchRestTemplate restTemplate, IndexCoordinates boundIndex) {
+		super(restTemplate.getElasticsearchConverter(), boundIndex);
+		this.restTemplate = restTemplate;
 	}
 
 	@Override
 	protected boolean doCreate(String indexName, @Nullable Document settings) {
 		CreateIndexRequest request = requestFactory.createIndexRequest(indexName, settings);
-		try {
-			return client.indices().create(request, RequestOptions.DEFAULT).isAcknowledged();
-		} catch (IOException e) {
-			throw new ElasticsearchException(
-					"Error for creating index: " + indexName + ", client: " + client.getLowLevelClient().getNodes(), e);
-		}
+		return restTemplate.execute(client -> client.indices().create(request, RequestOptions.DEFAULT).isAcknowledged());
 	}
 
 	@Override
@@ -92,11 +83,7 @@ class DefaultIndexOperations extends AbstractDefaultIndexOperations implements I
 
 		if (doExists(indexName)) {
 			DeleteIndexRequest request = new DeleteIndexRequest(indexName);
-			try {
-				return client.indices().delete(request, RequestOptions.DEFAULT).isAcknowledged();
-			} catch (IOException e) {
-				throw new ElasticsearchException("Error while deleting index request: " + request.toString(), e);
-			}
+			return restTemplate.execute(client -> client.indices().delete(request, RequestOptions.DEFAULT).isAcknowledged());
 		}
 		return false;
 	}
@@ -104,11 +91,7 @@ class DefaultIndexOperations extends AbstractDefaultIndexOperations implements I
 	@Override
 	protected boolean doExists(String indexName) {
 		GetIndexRequest request = new GetIndexRequest(indexName);
-		try {
-			return client.indices().exists(request, RequestOptions.DEFAULT);
-		} catch (IOException e) {
-			throw new ElasticsearchException("Error while for indexExists request: " + request.toString(), e);
-		}
+		return restTemplate.execute(client -> client.indices().exists(request, RequestOptions.DEFAULT));
 	}
 
 	@Override
@@ -117,11 +100,8 @@ class DefaultIndexOperations extends AbstractDefaultIndexOperations implements I
 		Assert.notNull(index, "No index defined for putMapping()");
 
 		PutMappingRequest request = requestFactory.putMappingRequest(index, mapping);
-		try {
-			return client.indices().putMapping(request, RequestOptions.DEFAULT).isAcknowledged();
-		} catch (IOException e) {
-			throw new ElasticsearchException("Failed to put mapping for " + index.getIndexName(), e);
-		}
+		return restTemplate
+				.execute(client -> client.indices().putMapping(request, RequestOptions.DEFAULT).isAcknowledged());
 	}
 
 	@Override
@@ -129,24 +109,19 @@ class DefaultIndexOperations extends AbstractDefaultIndexOperations implements I
 
 		Assert.notNull(index, "No index defined for getMapping()");
 
-		RestClient restClient = client.getLowLevelClient();
-		try {
+		return restTemplate.execute(client -> {
+			RestClient restClient = client.getLowLevelClient();
 			Request request = new Request("GET", '/' + index.getIndexName() + "/_mapping");
 			Response response = restClient.performRequest(request);
 			return convertMappingResponse(EntityUtils.toString(response.getEntity()));
-		} catch (Exception e) {
-			throw new ElasticsearchException("Error while getting mapping for indexName : " + index.getIndexName(), e);
-		}
+		});
 	}
 
 	@Override
 	protected boolean doAddAlias(AliasQuery query, IndexCoordinates index) {
 		IndicesAliasesRequest request = requestFactory.indicesAddAliasesRequest(query, index);
-		try {
-			return client.indices().updateAliases(request, RequestOptions.DEFAULT).isAcknowledged();
-		} catch (IOException e) {
-			throw new ElasticsearchException("failed to update aliases with request: " + request, e);
-		}
+		return restTemplate
+				.execute(client -> client.indices().updateAliases(request, RequestOptions.DEFAULT).isAcknowledged());
 	}
 
 	@Override
@@ -156,29 +131,23 @@ class DefaultIndexOperations extends AbstractDefaultIndexOperations implements I
 		Assert.notNull(query.getAliasName(), "No alias defined");
 
 		IndicesAliasesRequest indicesAliasesRequest = requestFactory.indicesRemoveAliasesRequest(query, index);
-		try {
-			return client.indices().updateAliases(indicesAliasesRequest, RequestOptions.DEFAULT).isAcknowledged();
-		} catch (IOException e) {
-			throw new ElasticsearchException(
-					"failed to update aliases with indicesRemoveAliasesRequest: " + indicesAliasesRequest, e);
-		}
+		return restTemplate.execute(
+				client -> client.indices().updateAliases(indicesAliasesRequest, RequestOptions.DEFAULT).isAcknowledged());
 	}
 
 	@Override
 	protected List<AliasMetaData> doQueryForAlias(String indexName) {
 		List<AliasMetaData> aliases = null;
-		RestClient restClient = client.getLowLevelClient();
-		Response response;
-		String aliasResponse;
+		return restTemplate.execute(client -> {
+			RestClient restClient = client.getLowLevelClient();
+			Response response;
+			String aliasResponse;
 
-		try {
 			response = restClient.performRequest(new Request("GET", '/' + indexName + "/_alias/*"));
 			aliasResponse = EntityUtils.toString(response.getEntity());
-		} catch (Exception e) {
-			throw new ElasticsearchException("Error while getting mapping for indexName : " + indexName, e);
-		}
 
-		return convertAliasResponse(aliasResponse);
+			return convertAliasResponse(aliasResponse);
+		});
 	}
 
 	@Override
@@ -190,14 +159,11 @@ class DefaultIndexOperations extends AbstractDefaultIndexOperations implements I
 				.indices(indexName) //
 				.includeDefaults(includeDefaults);
 
-		try {
-			GetSettingsResponse response = client.indices() //
-					.getSettings(request, RequestOptions.DEFAULT);
+		//
+		GetSettingsResponse response = restTemplate.execute(client -> client.indices() //
+				.getSettings(request, RequestOptions.DEFAULT));
 
-			return convertSettingsResponseToMap(response, indexName);
-		} catch (IOException e) {
-			throw new ElasticsearchException("failed to get settings for index: " + indexName, e);
-		}
+		return convertSettingsResponseToMap(response, indexName);
 	}
 
 	@Override
@@ -205,11 +171,8 @@ class DefaultIndexOperations extends AbstractDefaultIndexOperations implements I
 
 		Assert.notNull(index, "No index defined for refresh()");
 
-		try {
-			client.indices().refresh(refreshRequest(index.getIndexNames()), RequestOptions.DEFAULT);
-		} catch (IOException e) {
-			throw new ElasticsearchException("failed to refresh index: " + index, e);
-		}
+		restTemplate
+				.execute(client -> client.indices().refresh(refreshRequest(index.getIndexNames()), RequestOptions.DEFAULT));
 	}
 
 	// region Helper methods
@@ -225,7 +188,7 @@ class DefaultIndexOperations extends AbstractDefaultIndexOperations implements I
 
 			return result;
 		} catch (IOException e) {
-			throw new ElasticsearchException("Could not map alias response : " + mappingResponse, e);
+			throw new UncategorizedElasticsearchException("Could not map alias response : " + mappingResponse, e);
 		}
 	}
 
@@ -261,7 +224,7 @@ class DefaultIndexOperations extends AbstractDefaultIndexOperations implements I
 			}
 			return aliasMetaDataList;
 		} catch (IOException e) {
-			throw new ElasticsearchException("Could not map alias response : " + aliasResponse, e);
+			throw new UncategorizedElasticsearchException("Could not map alias response : " + aliasResponse, e);
 		}
 	}
 	// endregion

src/main/java/org/springframework/data/elasticsearch/core/DocumentOperations.java

@@ -16,7 +16,6 @@
 package org.springframework.data.elasticsearch.core;
 
 import java.util.List;
-import java.util.Optional;
 
 import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates;
 import org.springframework.data.elasticsearch.core.query.BulkOptions;
@@ -121,7 +120,7 @@ public interface DocumentOperations {
 	 * @param query the query defining the ids of the objects to get
 	 * @param clazz the type of the object to be returned
 	 * @param index the index(es) from which the objects are read.
-	 * @return list of objects
+	 * @return list of objects, contains null values for ids that are not found
 	 */
 	<T> List<T> multiGet(Query query, Class<T> clazz, IndexCoordinates index);
 
@@ -251,11 +250,22 @@ public interface DocumentOperations {
 	 * @param clazz the type of the object to be returned
 	 * @param index the index from which the object is read.
 	 * @return the found object
-	 * @deprecated since 4.0, use {@link #getById(String, Class, IndexCoordinates)}
+	 * @deprecated since 4.0, use {@link #get(String, Class, IndexCoordinates)}
 	 */
 	@Deprecated
 	@Nullable
 	<T> T get(GetQuery query, Class<T> clazz, IndexCoordinates index);
 
+	/**
+	 * Retrieves an object from an index.
+	 *
+	 * @param query the query defining the id of the object to get
+	 * @param clazz the type of the object to be returned
+	 * @return the found object
+	 * @deprecated since 4.0, use {@link #get(String, Class, IndexCoordinates)}
+	 */
+	@Deprecated
+	@Nullable
+	<T> T queryForObject(GetQuery query, Class<T> clazz);
 	// endregion
 }

src/main/java/org/springframework/data/elasticsearch/core/ElasticsearchExceptionTranslator.java

@@ -13,24 +13,37 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-
 package org.springframework.data.elasticsearch.core;
 
-import java.net.ConnectException;
+import java.io.IOException;
 import java.util.List;
 
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ElasticsearchStatusException;
+import org.elasticsearch.common.ValidationException;
+import org.elasticsearch.index.engine.VersionConflictEngineException;
+import org.elasticsearch.rest.RestStatus;
+
 import org.springframework.dao.DataAccessException;
 import org.springframework.dao.DataAccessResourceFailureException;
+import org.springframework.dao.DataIntegrityViolationException;
+import org.springframework.dao.OptimisticLockingFailureException;
 import org.springframework.dao.support.PersistenceExceptionTranslator;
 import org.springframework.data.elasticsearch.NoSuchIndexException;
+import org.springframework.data.elasticsearch.UncategorizedElasticsearchException;
 import org.springframework.util.CollectionUtils;
 import org.springframework.util.ObjectUtils;
 import org.springframework.util.StringUtils;
 
 /**
+ * Simple {@link PersistenceExceptionTranslator} for Elasticsearch. Convert the given runtime exception to an
+ * appropriate exception from the {@code org.springframework.dao} hierarchy. Return {@literal null} if no translation is
+ * appropriate: any other exception may have resulted from user code, and should not be translated.
+ * 
  * @author Christoph Strobl
+ * @author Peter-Josef Meisch
+ * @author Roman Puchkovskiy
+ * @author Mark Paluch
  * @since 3.2
  */
 public class ElasticsearchExceptionTranslator implements PersistenceExceptionTranslator {
@@ -38,6 +51,10 @@ public class ElasticsearchExceptionTranslator implements PersistenceExceptionTra
 	@Override
 	public DataAccessException translateExceptionIfPossible(RuntimeException ex) {
 
+		if (isSeqNoConflict(ex)) {
+			return new OptimisticLockingFailureException("Cannot index a document due to seq_no+primary_term conflict", ex);
+		}
+
 		if (ex instanceof ElasticsearchException) {
 
 			ElasticsearchException elasticsearchException = (ElasticsearchException) ex;
@@ -46,22 +63,52 @@ public class ElasticsearchExceptionTranslator implements PersistenceExceptionTra
 				return new NoSuchIndexException(ObjectUtils.nullSafeToString(elasticsearchException.getMetadata("es.index")),
 						ex);
 			}
+
+			return new UncategorizedElasticsearchException(ex.getMessage(), ex);
 		}
 
-		if (ex.getCause() instanceof ConnectException) {
+		if (ex instanceof ValidationException) {
+			return new DataIntegrityViolationException(ex.getMessage(), ex);
+		}
+
+		Throwable cause = ex.getCause();
+		if (cause instanceof IOException) {
 			return new DataAccessResourceFailureException(ex.getMessage(), ex);
 		}
 
 		return null;
 	}
 
+	private boolean isSeqNoConflict(Exception exception) {
+
+		if (exception instanceof ElasticsearchStatusException) {
+
+			ElasticsearchStatusException statusException = (ElasticsearchStatusException) exception;
+
+			return statusException.status() == RestStatus.CONFLICT && statusException.getMessage() != null
+					&& statusException.getMessage().contains("type=version_conflict_engine_exception")
+					&& statusException.getMessage().contains("version conflict, required seqNo");
+		}
+
+		if (exception instanceof VersionConflictEngineException) {
+
+			VersionConflictEngineException versionConflictEngineException = (VersionConflictEngineException) exception;
+
+			return versionConflictEngineException.getMessage() != null
+					&& versionConflictEngineException.getMessage().contains("version conflict, required seqNo");
+		}
+
+		return false;
+	}
+
 	private boolean indexAvailable(ElasticsearchException ex) {
 
 		List<String> metadata = ex.getMetadata("es.index_uuid");
 		if (metadata == null) {
 			if (ex instanceof ElasticsearchStatusException) {
-				return StringUtils.hasText(ObjectUtils.nullSafeToString(((ElasticsearchStatusException) ex).getIndex()));
+				return StringUtils.hasText(ObjectUtils.nullSafeToString(ex.getIndex()));
 			}
+			return false;
 		}
 		return !CollectionUtils.contains(metadata.iterator(), "_na_");
 	}

src/main/java/org/springframework/data/elasticsearch/core/ElasticsearchOperations.java

@@ -60,17 +60,17 @@ public interface ElasticsearchOperations extends DocumentOperations, SearchOpera
 	IndexCoordinates getIndexCoordinatesFor(Class<?> clazz);
 
 	// region IndexOperations
-	/**
-	 * Create an index for given indexName if it does not already exist.
-	 *
-	 * @param indexName the name of the index
-	 * @return {@literal true} if the index was created
-	 * @deprecated since 4.0, use {@link IndexOperations#create()}
-	 */
-	@Deprecated
-	default boolean createIndex(String indexName) {
-		return indexOps(IndexCoordinates.of(indexName)).create();
-	}
+/**
+ * Create an index for given indexName .
+ *
+ * @param indexName the name of the index
+ * @return {@literal true} if the index was created
+ * @deprecated since 4.0, use {@link IndexOperations#create()}
+ */
+@Deprecated
+default boolean createIndex(String indexName) {
+	return indexOps(IndexCoordinates.of(indexName)).create();
+}
 
 	/**
 	 * Create an index for given indexName and Settings.

src/main/java/org/springframework/data/elasticsearch/core/ElasticsearchRestTemplate.java

@@ -17,7 +17,7 @@ package org.springframework.data.elasticsearch.core;
 
 import java.io.IOException;
 import java.util.List;
-import java.util.Optional;
+import java.util.stream.Collectors;
 
 import org.elasticsearch.action.bulk.BulkRequest;
 import org.elasticsearch.action.delete.DeleteRequest;
@@ -38,9 +38,10 @@ import org.elasticsearch.client.RestHighLevelClient;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.reindex.DeleteByQueryRequest;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
 import org.elasticsearch.search.suggest.SuggestBuilder;
-import org.springframework.data.domain.Pageable;
-import org.springframework.data.elasticsearch.ElasticsearchException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.springframework.data.elasticsearch.core.convert.ElasticsearchConverter;
 import org.springframework.data.elasticsearch.core.document.DocumentAdapters;
 import org.springframework.data.elasticsearch.core.document.SearchDocumentResponse;
@@ -89,26 +90,32 @@ import org.springframework.util.Assert;
  */
 public class ElasticsearchRestTemplate extends AbstractElasticsearchTemplate {
 
+	private static final Logger LOGGER = LoggerFactory.getLogger(ElasticsearchRestTemplate.class);
+
 	private RestHighLevelClient client;
+	private ElasticsearchExceptionTranslator exceptionTranslator;
 
 	// region Initialization
 	public ElasticsearchRestTemplate(RestHighLevelClient client) {
-		this.client = client;
-		initialize(client, createElasticsearchConverter());
-	}
-
-	public ElasticsearchRestTemplate(RestHighLevelClient client, ElasticsearchConverter elasticsearchConverter) {
-		this.client = client;
-		initialize(client, elasticsearchConverter);
-	}
-
-	private void initialize(RestHighLevelClient client, ElasticsearchConverter elasticsearchConverter) {
 
 		Assert.notNull(client, "Client must not be null!");
 
 		this.client = client;
+		this.exceptionTranslator = new ElasticsearchExceptionTranslator();
+
+		initialize(createElasticsearchConverter());
+	}
+
+	public ElasticsearchRestTemplate(RestHighLevelClient client, ElasticsearchConverter elasticsearchConverter) {
+
+		Assert.notNull(client, "Client must not be null!");
+
+		this.client = client;
+		this.exceptionTranslator = new ElasticsearchExceptionTranslator();
+
 		initialize(elasticsearchConverter);
 	}
+
 	// endregion
 
 	// region IndexOperations
@@ -117,7 +124,7 @@ public class ElasticsearchRestTemplate extends AbstractElasticsearchTemplate {
 
 		Assert.notNull(clazz, "clazz must not be null");
 
-		return new DefaultIndexOperations(client, elasticsearchConverter, clazz);
+		return new DefaultIndexOperations(this, clazz);
 	}
 
 	@Override
@@ -125,37 +132,38 @@ public class ElasticsearchRestTemplate extends AbstractElasticsearchTemplate {
 
 		Assert.notNull(index, "index must not be null");
 
-		return new DefaultIndexOperations(client, elasticsearchConverter, index);
+		return new DefaultIndexOperations(this, index);
 	}
 	// endregion
 
 	// region DocumentOperations
 	@Override
 	public String index(IndexQuery query, IndexCoordinates index) {
-		IndexRequest request = requestFactory.indexRequest(query, index);
-		try {
-			String documentId = client.index(request, RequestOptions.DEFAULT).getId();
 
-			// We should call this because we are not going through a mapper.
-			if (query.getObject() != null) {
-				setPersistentEntityId(query.getObject(), documentId);
-			}
-			return documentId;
-		} catch (IOException e) {
-			throw new ElasticsearchException("Error while index for request: " + request.toString(), e);
+		maybeCallbackBeforeConvertWithQuery(query, index);
+
+		IndexRequest request = requestFactory.indexRequest(query, index);
+		String documentId = execute(client -> client.index(request, RequestOptions.DEFAULT).getId());
+
+		// We should call this because we are not going through a mapper.
+		Object queryObject = query.getObject();
+		if (queryObject != null) {
+			setPersistentEntityId(queryObject, documentId);
 		}
+
+		maybeCallbackAfterSaveWithQuery(query, index);
+
+		return documentId;
 	}
 
 	@Override
 	@Nullable
 	public <T> T get(String id, Class<T> clazz, IndexCoordinates index) {
 		GetRequest request = requestFactory.getRequest(id, index);
-		try {
-			GetResponse response = client.get(request, RequestOptions.DEFAULT);
-			return elasticsearchConverter.mapDocument(DocumentAdapters.from(response), clazz);
-		} catch (IOException e) {
-			throw new ElasticsearchException("Error while getting for request: " + request.toString(), e);
-		}
+		GetResponse response = execute(client -> client.get(request, RequestOptions.DEFAULT));
+
+		DocumentCallback<T> callback = new ReadDocumentCallback<>(elasticsearchConverter, clazz, index);
+		return callback.doWith(DocumentAdapters.from(response));
 	}
 
 	@Override
@@ -165,22 +173,17 @@ public class ElasticsearchRestTemplate extends AbstractElasticsearchTemplate {
 		Assert.notEmpty(query.getIds(), "No Id define for Query");
 
 		MultiGetRequest request = requestFactory.multiGetRequest(query, index);
-		try {
-			MultiGetResponse result = client.mget(request, RequestOptions.DEFAULT);
-			return elasticsearchConverter.mapDocuments(DocumentAdapters.from(result), clazz);
-		} catch (IOException e) {
-			throw new ElasticsearchException("Error while multiget for request: " + request.toString(), e);
-		}
+		MultiGetResponse result = execute(client -> client.mget(request, RequestOptions.DEFAULT));
+
+		DocumentCallback<T> callback = new ReadDocumentCallback<>(elasticsearchConverter, clazz, index);
+		return DocumentAdapters.from(result).stream().map(callback::doWith).collect(Collectors.toList());
 	}
 
 	@Override
 	protected boolean doExists(String id, IndexCoordinates index) {
 		GetRequest request = requestFactory.getRequest(id, index);
-		try {
-			return client.get(request, RequestOptions.DEFAULT).isExists();
-		} catch (IOException e) {
-			throw new ElasticsearchException("Error while getting for request: " + request.toString(), e);
-		}
+		request.fetchSourceContext(FetchSourceContext.DO_NOT_FETCH_SOURCE);
+		return execute(client -> client.get(request, RequestOptions.DEFAULT).isExists());
 	}
 
 	@Override
@@ -207,54 +210,38 @@ public class ElasticsearchRestTemplate extends AbstractElasticsearchTemplate {
 		Assert.notNull(id, "id must not be null");
 		Assert.notNull(index, "index must not be null");
 
-		DeleteRequest request = new DeleteRequest(index.getIndexName(), elasticsearchConverter.convertId(id));
-		try {
-			return client.delete(request, RequestOptions.DEFAULT).getId();
-		} catch (IOException e) {
-			throw new ElasticsearchException("Error while deleting item request: " + request.toString(), e);
-		}
+		DeleteRequest request = requestFactory.deleteRequest(elasticsearchConverter.convertId(id), index);
+		return execute(client -> client.delete(request, RequestOptions.DEFAULT).getId());
 	}
 
 	@Override
 	public void delete(Query query, Class<?> clazz, IndexCoordinates index) {
 		DeleteByQueryRequest deleteByQueryRequest = requestFactory.deleteByQueryRequest(query, clazz, index);
-		try {
-			client.deleteByQuery(deleteByQueryRequest, RequestOptions.DEFAULT);
-		} catch (IOException e) {
-			throw new ElasticsearchException("Error for delete request: " + deleteByQueryRequest.toString(), e);
-		}
+		execute(client -> client.deleteByQuery(deleteByQueryRequest, RequestOptions.DEFAULT));
 	}
 
 	@Override
 	@Deprecated
 	public void delete(DeleteQuery deleteQuery, IndexCoordinates index) {
 		DeleteByQueryRequest deleteByQueryRequest = requestFactory.deleteByQueryRequest(deleteQuery, index);
-		try {
-			client.deleteByQuery(deleteByQueryRequest, RequestOptions.DEFAULT);
-		} catch (IOException e) {
-			throw new ElasticsearchException("Error for delete request: " + deleteByQueryRequest.toString(), e);
-		}
+		execute(client -> client.deleteByQuery(deleteByQueryRequest, RequestOptions.DEFAULT));
 	}
 
 	@Override
 	public UpdateResponse update(UpdateQuery query, IndexCoordinates index) {
 		UpdateRequest request = requestFactory.updateRequest(query, index);
-		try {
-			org.elasticsearch.action.update.UpdateResponse updateResponse = client.update(request, RequestOptions.DEFAULT);
-			UpdateResponse.Result result = UpdateResponse.Result.valueOf(updateResponse.getResult().name());
-			return new UpdateResponse(result);
-		} catch (IOException e) {
-			throw new ElasticsearchException("Error while update for request: " + request.toString(), e);
-		}
+		UpdateResponse.Result result = UpdateResponse.Result
+				.valueOf(execute(client -> client.update(request, RequestOptions.DEFAULT)).getResult().name());
+		return new UpdateResponse(result);
 	}
 
 	private List<String> doBulkOperation(List<?> queries, BulkOptions bulkOptions, IndexCoordinates index) {
+		maybeCallbackBeforeConvertWithQueries(queries, index);
 		BulkRequest bulkRequest = requestFactory.bulkRequest(queries, bulkOptions, index);
-		try {
-			return checkForBulkOperationFailure(client.bulk(bulkRequest, RequestOptions.DEFAULT));
-		} catch (IOException e) {
-			throw new ElasticsearchException("Error while bulk for request: " + bulkRequest.toString(), e);
-		}
+		List<String> ids = checkForBulkOperationFailure(
+				execute(client -> client.bulk(bulkRequest, RequestOptions.DEFAULT)));
+		maybeCallbackAfterSaveWithQueries(queries, index);
+		return ids;
 	}
 	// endregion
 
@@ -272,64 +259,57 @@ public class ElasticsearchRestTemplate extends AbstractElasticsearchTemplate {
 
 		searchRequest.source().size(0);
 
-		try {
-			return SearchHitsUtil.getTotalCount(client.search(searchRequest, RequestOptions.DEFAULT).getHits());
-		} catch (IOException e) {
-			throw new ElasticsearchException("Error for search request: " + searchRequest.toString(), e);
-		}
+		return SearchHitsUtil
+				.getTotalCount(execute(client -> client.search(searchRequest, RequestOptions.DEFAULT).getHits()));
 	}
 
 	@Override
 	public <T> SearchHits<T> search(Query query, Class<T> clazz, IndexCoordinates index) {
 		SearchRequest searchRequest = requestFactory.searchRequest(query, clazz, index);
-		SearchResponse response;
-		try {
-			response = client.search(searchRequest, RequestOptions.DEFAULT);
-		} catch (IOException e) {
-			throw new ElasticsearchException("Error for search request: " + searchRequest.toString(), e);
-		}
-		return elasticsearchConverter.read(clazz, SearchDocumentResponse.from(response));
+		SearchResponse response = execute(client -> client.search(searchRequest, RequestOptions.DEFAULT));
+
+		SearchDocumentResponseCallback<SearchHits<T>> callback = new ReadSearchDocumentResponseCallback<>(clazz, index);
+		return callback.doWith(SearchDocumentResponse.from(response));
 	}
 
 	@Override
-	public <T> ScrolledPage<SearchHit<T>> searchScrollStart(long scrollTimeInMillis, Query query, Class<T> clazz,
+	public <T> SearchScrollHits<T> searchScrollStart(long scrollTimeInMillis, Query query, Class<T> clazz,
 			IndexCoordinates index) {
 
-		Assert.notNull(query.getPageable(), "Query.pageable is required for scan & scroll");
+		Assert.notNull(query.getPageable(), "pageable of query must not be null.");
 
 		SearchRequest searchRequest = requestFactory.searchRequest(query, clazz, index);
 		searchRequest.scroll(TimeValue.timeValueMillis(scrollTimeInMillis));
 
-		try {
-			SearchResponse result = client.search(searchRequest, RequestOptions.DEFAULT);
-			return elasticsearchConverter.mapResults(SearchDocumentResponse.from(result), clazz, null);
-		} catch (IOException e) {
-			throw new ElasticsearchException("Error for search request with scroll: " + searchRequest.toString(), e);
-		}
+		SearchResponse response = execute(client -> client.search(searchRequest, RequestOptions.DEFAULT));
+
+		SearchDocumentResponseCallback<SearchScrollHits<T>> callback = new ReadSearchScrollDocumentResponseCallback<>(clazz,
+				index);
+		return callback.doWith(SearchDocumentResponse.from(response));
 	}
 
 	@Override
-	public <T> ScrolledPage<SearchHit<T>> searchScrollContinue(@Nullable String scrollId, long scrollTimeInMillis,
-			Class<T> clazz) {
+	public <T> SearchScrollHits<T> searchScrollContinue(@Nullable String scrollId, long scrollTimeInMillis,
+			Class<T> clazz, IndexCoordinates index) {
+
 		SearchScrollRequest request = new SearchScrollRequest(scrollId);
 		request.scroll(TimeValue.timeValueMillis(scrollTimeInMillis));
-		SearchResponse response;
-		try {
-			response = client.searchScroll(request, RequestOptions.DEFAULT);
-		} catch (IOException e) {
-			throw new ElasticsearchException("Error for search request with scroll: " + request.toString(), e);
-		}
-		return elasticsearchConverter.mapResults(SearchDocumentResponse.from(response), clazz, Pageable.unpaged());
+
+		SearchResponse response = execute(client -> client.scroll(request, RequestOptions.DEFAULT));
+
+		SearchDocumentResponseCallback<SearchScrollHits<T>> callback = //
+				new ReadSearchScrollDocumentResponseCallback<>(clazz, index);
+		return callback.doWith(SearchDocumentResponse.from(response));
 	}
 
 	@Override
-	public void searchScrollClear(String scrollId) {
-		ClearScrollRequest request = new ClearScrollRequest();
-		request.addScrollId(scrollId);
+	public void searchScrollClear(List<String> scrollIds) {
 		try {
-			client.clearScroll(request, RequestOptions.DEFAULT);
-		} catch (IOException e) {
-			throw new ElasticsearchException("Error for search request with scroll: " + request.toString(), e);
+			ClearScrollRequest request = new ClearScrollRequest();
+			request.scrollIds(scrollIds);
+			execute(client -> client.clearScroll(request, RequestOptions.DEFAULT));
+		} catch (Exception e) {
+			LOGGER.warn("Could not clear scroll: {}", e.getMessage());
 		}
 	}
 
@@ -339,26 +319,76 @@ public class ElasticsearchRestTemplate extends AbstractElasticsearchTemplate {
 		SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
 		sourceBuilder.suggest(suggestion);
 		searchRequest.source(sourceBuilder);
-
-		try {
-			return client.search(searchRequest, RequestOptions.DEFAULT);
-		} catch (IOException e) {
-			throw new ElasticsearchException("Could not execute search request : " + searchRequest.toString(), e);
-		}
-
+		return execute(client -> client.search(searchRequest, RequestOptions.DEFAULT));
 	}
 
 	@Override
 	protected MultiSearchResponse.Item[] getMultiSearchResult(MultiSearchRequest request) {
-		MultiSearchResponse response;
-		try {
-			response = client.multiSearch(request, RequestOptions.DEFAULT);
-		} catch (IOException e) {
-			throw new ElasticsearchException("Error for search request: " + request.toString(), e);
-		}
+		MultiSearchResponse response = execute(client -> client.multiSearch(request, RequestOptions.DEFAULT));
 		MultiSearchResponse.Item[] items = response.getResponses();
 		Assert.isTrue(items.length == request.requests().size(), "Response should has same length with queries");
 		return items;
 	}
 	// endregion
+
+	// region ClientCallback
+	/**
+	 * Callback interface to be used with {@link #execute(ClientCallback)} for operating directly on
+	 * {@link RestHighLevelClient}.
+	 *
+	 * @since 4.0
+	 */
+	@FunctionalInterface
+	public interface ClientCallback<T> {
+		T doWithClient(RestHighLevelClient client) throws IOException;
+	}
+
+	/**
+	 * Execute a callback with the {@link RestHighLevelClient}
+	 *
+	 * @param callback the callback to execute, must not be {@literal null}
+	 * @param <T> the type returned from the callback
+	 * @return the callback result
+	 * @since 4.0
+	 */
+	public <T> T execute(ClientCallback<T> callback) {
+
+		Assert.notNull(callback, "callback must not be null");
+
+		try {
+			return callback.doWithClient(client);
+		} catch (IOException | RuntimeException e) {
+			throw translateException(e);
+		}
+	}
+
+	/**
+	 * translates an Exception if possible. Exceptions that are no {@link RuntimeException}s are wrapped in a
+	 * RuntimeException
+	 *
+	 * @param exception the Exception to map
+	 * @return the potentially translated RuntimeException.
+	 * @since 4.0
+	 */
+	private RuntimeException translateException(Exception exception) {
+
+		RuntimeException runtimeException = exception instanceof RuntimeException ? (RuntimeException) exception
+				: new RuntimeException(exception.getMessage(), exception);
+		RuntimeException potentiallyTranslatedException = exceptionTranslator
+				.translateExceptionIfPossible(runtimeException);
+
+		return potentiallyTranslatedException != null ? potentiallyTranslatedException : runtimeException;
+	}
+
+	// endregion
+
+	// region helper methods
+	@Override
+	protected String getClusterVersion() {
+		try {
+			return execute(client -> client.info(RequestOptions.DEFAULT)).getVersion().getNumber();
+		} catch (Exception ignored) {}
+		return null;
+	}
+	// endregion
 }

src/main/java/org/springframework/data/elasticsearch/core/ElasticsearchTemplate.java

@@ -16,14 +16,18 @@
 package org.springframework.data.elasticsearch.core;
 
 import java.util.List;
-import java.util.Optional;
+import java.util.stream.Collectors;
 
 import org.elasticsearch.action.ActionFuture;
+import org.elasticsearch.action.admin.cluster.node.info.NodesInfoAction;
+import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequestBuilder;
+import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse;
 import org.elasticsearch.action.bulk.BulkRequestBuilder;
 import org.elasticsearch.action.get.GetRequestBuilder;
 import org.elasticsearch.action.get.GetResponse;
 import org.elasticsearch.action.get.MultiGetRequestBuilder;
 import org.elasticsearch.action.index.IndexRequestBuilder;
+import org.elasticsearch.action.index.IndexResponse;
 import org.elasticsearch.action.search.MultiSearchRequest;
 import org.elasticsearch.action.search.MultiSearchResponse;
 import org.elasticsearch.action.search.SearchRequestBuilder;
@@ -34,8 +38,8 @@ import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.search.suggest.SuggestBuilder;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import org.springframework.data.domain.Pageable;
 import org.springframework.data.elasticsearch.core.convert.ElasticsearchConverter;
+import org.springframework.data.elasticsearch.core.document.Document;
 import org.springframework.data.elasticsearch.core.document.DocumentAdapters;
 import org.springframework.data.elasticsearch.core.document.SearchDocumentResponse;
 import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates;
@@ -75,16 +79,20 @@ import org.springframework.util.Assert;
  * @author Martin Choraine
  * @author Farid Azaza
  * @author Gyula Attila Csorogi
+ * @author Roman Puchkovskiy
  * @deprecated as of 4.0
  */
 @Deprecated
 public class ElasticsearchTemplate extends AbstractElasticsearchTemplate {
 	private static final Logger QUERY_LOGGER = LoggerFactory
 			.getLogger("org.springframework.data.elasticsearch.core.QUERY");
+	private static final Logger LOGGER = LoggerFactory.getLogger(ElasticsearchTemplate.class);
 
 	private Client client;
 	@Nullable private String searchTimeout;
 
+	private final ElasticsearchExceptionTranslator exceptionTranslator = new ElasticsearchExceptionTranslator();
+
 	// region Initialization
 	public ElasticsearchTemplate(Client client) {
 		this.client = client;
@@ -137,12 +145,27 @@ public class ElasticsearchTemplate extends AbstractElasticsearchTemplate {
 	// region DocumentOperations
 	@Override
 	public String index(IndexQuery query, IndexCoordinates index) {
+
+		maybeCallbackBeforeConvertWithQuery(query, index);
+
 		IndexRequestBuilder indexRequestBuilder = requestFactory.indexRequestBuilder(client, query, index);
-		String documentId = indexRequestBuilder.execute().actionGet().getId();
-		// We should call this because we are not going through a mapper.
-		if (query.getObject() != null) {
-			setPersistentEntityId(query.getObject(), documentId);
+		ActionFuture<IndexResponse> future = indexRequestBuilder.execute();
+		IndexResponse response;
+		try {
+			response = future.actionGet();
+		} catch (RuntimeException e) {
+			throw translateException(e);
 		}
+		String documentId = response.getId();
+
+		// We should call this because we are not going through a mapper.
+		Object queryObject = query.getObject();
+		if (queryObject != null) {
+			setPersistentEntityId(queryObject, documentId);
+		}
+
+		maybeCallbackAfterSaveWithQuery(query, index);
+
 		return documentId;
 	}
 
@@ -151,23 +174,28 @@ public class ElasticsearchTemplate extends AbstractElasticsearchTemplate {
 	public <T> T get(String id, Class<T> clazz, IndexCoordinates index) {
 		GetRequestBuilder getRequestBuilder = requestFactory.getRequestBuilder(client, id, index);
 		GetResponse response = getRequestBuilder.execute().actionGet();
-		return elasticsearchConverter.mapDocument(DocumentAdapters.from(response), clazz);
+
+		DocumentCallback<T> callback = new ReadDocumentCallback<>(elasticsearchConverter, clazz, index);
+		return callback.doWith(DocumentAdapters.from(response));
 	}
 
 	@Override
 	public <T> List<T> multiGet(Query query, Class<T> clazz, IndexCoordinates index) {
 
 		Assert.notNull(index, "index must not be null");
-		Assert.notEmpty(query.getIds(), "No Id define for Query");
+		Assert.notEmpty(query.getIds(), "No Ids defined for Query");
 
 		MultiGetRequestBuilder builder = requestFactory.multiGetRequestBuilder(client, query, index);
 
-		return elasticsearchConverter.mapDocuments(DocumentAdapters.from(builder.execute().actionGet()), clazz);
+		DocumentCallback<T> callback = new ReadDocumentCallback<>(elasticsearchConverter, clazz, index);
+		List<Document> documents = DocumentAdapters.from(builder.execute().actionGet());
+		return documents.stream().map(callback::doWith).collect(Collectors.toList());
 	}
 
 	@Override
 	protected boolean doExists(String id, IndexCoordinates index) {
 		GetRequestBuilder getRequestBuilder = requestFactory.getRequestBuilder(client, id, index);
+		getRequestBuilder.setFetchSource(false);
 		return getRequestBuilder.execute().actionGet().isExists();
 	}
 
@@ -177,7 +205,11 @@ public class ElasticsearchTemplate extends AbstractElasticsearchTemplate {
 		Assert.notNull(queries, "List of IndexQuery must not be null");
 		Assert.notNull(bulkOptions, "BulkOptions must not be null");
 
-		return doBulkOperation(queries, bulkOptions, index);
+		List<String> ids = doBulkOperation(queries, bulkOptions, index);
+
+		maybeCallbackAfterSaveWithQueries(queries, index);
+
+		return ids;
 	}
 
 	@Override
@@ -224,6 +256,7 @@ public class ElasticsearchTemplate extends AbstractElasticsearchTemplate {
 	}
 
 	private List<String> doBulkOperation(List<?> queries, BulkOptions bulkOptions, IndexCoordinates index) {
+		maybeCallbackBeforeConvertWithQueries(queries, index);
 		BulkRequestBuilder bulkRequest = requestFactory.bulkRequestBuilder(client, queries, bulkOptions, index);
 		return checkForBulkOperationFailure(bulkRequest.execute().actionGet());
 	}
@@ -249,31 +282,52 @@ public class ElasticsearchTemplate extends AbstractElasticsearchTemplate {
 	public <T> SearchHits<T> search(Query query, Class<T> clazz, IndexCoordinates index) {
 		SearchRequestBuilder searchRequestBuilder = requestFactory.searchRequestBuilder(client, query, clazz, index);
 		SearchResponse response = getSearchResponse(searchRequestBuilder);
-		return elasticsearchConverter.read(clazz, SearchDocumentResponse.from(response));
+
+		SearchDocumentResponseCallback<SearchHits<T>> callback = new ReadSearchDocumentResponseCallback<>(clazz, index);
+		return callback.doWith(SearchDocumentResponse.from(response));
 	}
 
 	@Override
-	public <T> ScrolledPage<SearchHit<T>> searchScrollStart(long scrollTimeInMillis, Query query, Class<T> clazz,
+	public <T> SearchScrollHits<T> searchScrollStart(long scrollTimeInMillis, Query query, Class<T> clazz,
 			IndexCoordinates index) {
-		Assert.notNull(query.getPageable(), "Query.pageable is required for scan & scroll");
 
-		SearchRequestBuilder searchRequestBuilder = requestFactory.searchRequestBuilder(client, query, clazz, index);
-		searchRequestBuilder.setScroll(TimeValue.timeValueMillis(scrollTimeInMillis));
-		SearchResponse response = getSearchResponse(searchRequestBuilder);
-		return elasticsearchConverter.mapResults(SearchDocumentResponse.from(response), clazz, null);
+		Assert.notNull(query.getPageable(), "pageable of query must not be null.");
+
+		ActionFuture<SearchResponse> action = requestFactory //
+				.searchRequestBuilder(client, query, clazz, index) //
+				.setScroll(TimeValue.timeValueMillis(scrollTimeInMillis)) //
+				.execute();
+
+		SearchResponse response = getSearchResponseWithTimeout(action);
+
+		SearchDocumentResponseCallback<SearchScrollHits<T>> callback = new ReadSearchScrollDocumentResponseCallback<>(clazz,
+				index);
+		return callback.doWith(SearchDocumentResponse.from(response));
 	}
 
 	@Override
-	public <T> ScrolledPage<SearchHit<T>> searchScrollContinue(@Nullable String scrollId, long scrollTimeInMillis,
-			Class<T> clazz) {
-		SearchResponse response = getSearchResponseWithTimeout(
-				client.prepareSearchScroll(scrollId).setScroll(TimeValue.timeValueMillis(scrollTimeInMillis)).execute());
-		return elasticsearchConverter.mapResults(SearchDocumentResponse.from(response), clazz, Pageable.unpaged());
+	public <T> SearchScrollHits<T> searchScrollContinue(@Nullable String scrollId, long scrollTimeInMillis,
+			Class<T> clazz, IndexCoordinates index) {
+
+		ActionFuture<SearchResponse> action = client //
+				.prepareSearchScroll(scrollId) //
+				.setScroll(TimeValue.timeValueMillis(scrollTimeInMillis)) //
+				.execute();
+
+		SearchResponse response = getSearchResponseWithTimeout(action);
+
+		SearchDocumentResponseCallback<SearchScrollHits<T>> callback = new ReadSearchScrollDocumentResponseCallback<>(clazz,
+				index);
+		return callback.doWith(SearchDocumentResponse.from(response));
 	}
 
 	@Override
-	public void searchScrollClear(String scrollId) {
-		client.prepareClearScroll().addScrollId(scrollId).execute().actionGet();
+	public void searchScrollClear(List<String> scrollIds) {
+		try {
+			client.prepareClearScroll().setScrollIds(scrollIds).execute().actionGet();
+		} catch (Exception e) {
+			LOGGER.warn("Could not clear scroll: {}", e.getMessage());
+		}
 	}
 
 	@Override
@@ -302,4 +356,41 @@ public class ElasticsearchTemplate extends AbstractElasticsearchTemplate {
 		return searchTimeout == null ? response.actionGet() : response.actionGet(searchTimeout);
 	}
 	// endregion
+
+	// region helper methods
+	@Override
+	protected String getClusterVersion() {
+
+		try {
+			NodesInfoResponse nodesInfoResponse = client.admin().cluster()
+					.nodesInfo(new NodesInfoRequestBuilder(client, NodesInfoAction.INSTANCE).request()).actionGet();
+			if (!nodesInfoResponse.getNodes().isEmpty()) {
+				return nodesInfoResponse.getNodes().get(0).getVersion().toString();
+			}
+		} catch (Exception ignored) {}
+		return null;
+	}
+
+	public Client getClient() {
+		return client;
+	}
+	// endregion
+
+	/**
+	 * translates an Exception if possible. Exceptions that are no {@link RuntimeException}s are wrapped in a
+	 * RuntimeException
+	 *
+	 * @param exception the Exception to map
+	 * @return the potentially translated RuntimeException.
+	 * @since 4.0
+	 */
+	private RuntimeException translateException(Exception exception) {
+
+		RuntimeException runtimeException = exception instanceof RuntimeException ? (RuntimeException) exception
+				: new RuntimeException(exception.getMessage(), exception);
+		RuntimeException potentiallyTranslatedException = exceptionTranslator
+				.translateExceptionIfPossible(runtimeException);
+
+		return potentiallyTranslatedException != null ? potentiallyTranslatedException : runtimeException;
+	}
 }

src/main/java/org/springframework/data/elasticsearch/core/EntityOperations.java

@@ -15,16 +15,13 @@
  */
 package org.springframework.data.elasticsearch.core;
 
-import lombok.AccessLevel;
-import lombok.NonNull;
-import lombok.RequiredArgsConstructor;
-
 import java.util.Map;
 
 import org.springframework.core.convert.ConversionService;
 import org.springframework.data.elasticsearch.core.mapping.ElasticsearchPersistentEntity;
 import org.springframework.data.elasticsearch.core.mapping.ElasticsearchPersistentProperty;
 import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates;
+import org.springframework.data.elasticsearch.core.query.SeqNoPrimaryTerm;
 import org.springframework.data.mapping.IdentifierAccessor;
 import org.springframework.data.mapping.PersistentPropertyAccessor;
 import org.springframework.data.mapping.context.MappingContext;
@@ -39,6 +36,7 @@ import org.springframework.util.StringUtils;
  * @author Mark Paluch
  * @author Christoph Strobl
  * @author Peter-Josef Meisch
+ * @author Roman Puchkovskiy
  * @since 3.2
  */
 class EntityOperations {
@@ -46,11 +44,14 @@ class EntityOperations {
 	private static final String ID_FIELD = "id";
 
 	public EntityOperations(
-			@NonNull MappingContext<? extends ElasticsearchPersistentEntity<?>, ElasticsearchPersistentProperty> context) {
+			MappingContext<? extends ElasticsearchPersistentEntity<?>, ElasticsearchPersistentProperty> context) {
+
+		Assert.notNull(context, "context must not be null");
+
 		this.context = context;
 	}
 
-	private final @NonNull MappingContext<? extends ElasticsearchPersistentEntity<?>, ElasticsearchPersistentProperty> context;
+	private final MappingContext<? extends ElasticsearchPersistentEntity<?>, ElasticsearchPersistentProperty> context;
 
 	/**
 	 * Creates a new {@link Entity} for the given bean.
@@ -257,6 +258,22 @@ class EntityOperations {
 		@Override
 		@Nullable
 		Number getVersion();
+
+		/**
+		 * Returns whether there is a property with type SeqNoPrimaryTerm in this entity.
+		 *
+		 * @return true if there is SeqNoPrimaryTerm property
+		 * @since 4.0
+		 */
+		boolean hasSeqNoPrimaryTerm();
+
+		/**
+		 * Returns SeqNoPropertyTerm for this entity.
+		 *
+		 * @return SeqNoPrimaryTerm, may be {@literal null}
+		 */
+		@Nullable
+		SeqNoPrimaryTerm getSeqNoPrimaryTerm();
 	}
 
 	/**
@@ -264,9 +281,15 @@ class EntityOperations {
 	 * @author Christoph Strobl
 	 * @since 3.2
 	 */
-	@RequiredArgsConstructor
 	private static class MapBackedEntity<T extends Map<String, Object>> implements AdaptibleEntity<T> {
 
+		public MapBackedEntity(T map) {
+
+			Assert.notNull(map, "map must not be null");
+
+			this.map = map;
+		}
+
 		private final T map;
 
 		/*
@@ -328,6 +351,16 @@ class EntityOperations {
 			return null;
 		}
 
+		@Override
+		public boolean hasSeqNoPrimaryTerm() {
+			return false;
+		}
+
+		@Override
+		public SeqNoPrimaryTerm getSeqNoPrimaryTerm() {
+			return null;
+		}
+
 		/*
 		 * (non-Javadoc)
 		 * @see org.springframework.data.elasticsearch.core.EntityOperations.AdaptibleEntity#incrementVersion()
@@ -406,13 +439,24 @@ class EntityOperations {
 	 * @param <T>
 	 * @since 3.2
 	 */
-	@RequiredArgsConstructor(access = AccessLevel.PROTECTED)
 	private static class MappedEntity<T> implements Entity<T> {
 
 		private final ElasticsearchPersistentEntity<?> entity;
 		private final IdentifierAccessor idAccessor;
 		private final PersistentPropertyAccessor<T> propertyAccessor;
 
+		private MappedEntity(ElasticsearchPersistentEntity<?> entity, IdentifierAccessor idAccessor,
+				PersistentPropertyAccessor<T> propertyAccessor) {
+
+			Assert.notNull(entity, "entity must not ne null");
+			Assert.notNull(idAccessor, "idAccessor must not ne null");
+			Assert.notNull(propertyAccessor, "propertyAccessor must not ne null");
+
+			this.entity = entity;
+			this.idAccessor = idAccessor;
+			this.propertyAccessor = propertyAccessor;
+		}
+
 		private static <T> MappedEntity<T> of(T bean,
 				MappingContext<? extends ElasticsearchPersistentEntity<?>, ElasticsearchPersistentProperty> context) {
 
@@ -572,6 +616,19 @@ class EntityOperations {
 			return propertyAccessor.getProperty(versionProperty, Number.class);
 		}
 
+		@Override
+		public boolean hasSeqNoPrimaryTerm() {
+			return entity.hasSeqNoPrimaryTermProperty();
+		}
+
+		@Override
+		public SeqNoPrimaryTerm getSeqNoPrimaryTerm() {
+
+			ElasticsearchPersistentProperty seqNoPrimaryTermProperty = entity.getRequiredSeqNoPrimaryTermProperty();
+
+			return propertyAccessor.getProperty(seqNoPrimaryTermProperty, SeqNoPrimaryTerm.class);
+		}
+
 		/* 
 		 * (non-Javadoc)
 		 * @see org.springframework.data.elasticsearch.core.EntityOperations.AdaptibleEntity#initializeVersionProperty()

src/main/java/org/springframework/data/elasticsearch/core/ReactiveElasticsearchTemplate.java

@@ -15,16 +15,12 @@
  */
 package org.springframework.data.elasticsearch.core;
 
-import static org.elasticsearch.index.VersionType.*;
-
 import reactor.core.publisher.Flux;
 import reactor.core.publisher.Mono;
+import reactor.util.function.Tuple2;
 
-import java.util.ArrayList;
 import java.util.Collection;
-import java.util.Collections;
 import java.util.HashMap;
-import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
@@ -41,24 +37,20 @@ import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.action.support.WriteRequest.RefreshPolicy;
-import org.elasticsearch.client.Requests;
-import org.elasticsearch.client.core.CountRequest;
 import org.elasticsearch.index.get.GetResult;
-import org.elasticsearch.index.query.QueryBuilder;
-import org.elasticsearch.index.query.QueryBuilders;
-import org.elasticsearch.index.query.WrapperQueryBuilder;
 import org.elasticsearch.index.reindex.BulkByScrollResponse;
 import org.elasticsearch.index.reindex.DeleteByQueryRequest;
-import org.elasticsearch.search.builder.SearchSourceBuilder;
-import org.elasticsearch.search.sort.FieldSortBuilder;
-import org.elasticsearch.search.sort.SortBuilders;
-import org.elasticsearch.search.sort.SortOrder;
+import org.elasticsearch.search.aggregations.Aggregation;
 import org.reactivestreams.Publisher;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import org.springframework.data.domain.Sort;
-import org.springframework.data.elasticsearch.ElasticsearchException;
+import org.springframework.beans.BeansException;
+import org.springframework.context.ApplicationContext;
+import org.springframework.context.ApplicationContextAware;
+import org.springframework.data.convert.EntityReader;
+import org.springframework.data.elasticsearch.BulkFailureException;
 import org.springframework.data.elasticsearch.NoSuchIndexException;
+import org.springframework.data.elasticsearch.UncategorizedElasticsearchException;
 import org.springframework.data.elasticsearch.client.reactive.ReactiveElasticsearchClient;
 import org.springframework.data.elasticsearch.core.EntityOperations.AdaptibleEntity;
 import org.springframework.data.elasticsearch.core.EntityOperations.Entity;
@@ -67,6 +59,9 @@ import org.springframework.data.elasticsearch.core.convert.MappingElasticsearchC
 import org.springframework.data.elasticsearch.core.document.Document;
 import org.springframework.data.elasticsearch.core.document.DocumentAdapters;
 import org.springframework.data.elasticsearch.core.document.SearchDocument;
+import org.springframework.data.elasticsearch.core.event.ReactiveAfterConvertCallback;
+import org.springframework.data.elasticsearch.core.event.ReactiveAfterSaveCallback;
+import org.springframework.data.elasticsearch.core.event.ReactiveBeforeConvertCallback;
 import org.springframework.data.elasticsearch.core.mapping.ElasticsearchPersistentEntity;
 import org.springframework.data.elasticsearch.core.mapping.ElasticsearchPersistentProperty;
 import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates;
@@ -74,12 +69,14 @@ import org.springframework.data.elasticsearch.core.mapping.SimpleElasticsearchMa
 import org.springframework.data.elasticsearch.core.query.BulkOptions;
 import org.springframework.data.elasticsearch.core.query.CriteriaQuery;
 import org.springframework.data.elasticsearch.core.query.IndexQuery;
-import org.springframework.data.elasticsearch.core.query.NativeSearchQuery;
 import org.springframework.data.elasticsearch.core.query.Query;
-import org.springframework.data.elasticsearch.core.query.StringQuery;
+import org.springframework.data.elasticsearch.core.query.SeqNoPrimaryTerm;
 import org.springframework.data.elasticsearch.core.query.UpdateQuery;
+import org.springframework.data.elasticsearch.support.VersionInfo;
+import org.springframework.data.mapping.callback.ReactiveEntityCallbacks;
 import org.springframework.data.mapping.context.MappingContext;
 import org.springframework.http.HttpStatus;
+import org.springframework.lang.NonNull;
 import org.springframework.lang.Nullable;
 import org.springframework.util.Assert;
 
@@ -91,9 +88,11 @@ import org.springframework.util.Assert;
  * @author Peter-Josef Meisch
  * @author Mathias Teier
  * @author Aleksei Arsenev
+ * @author Roman Puchkovskiy
+ * @author Russell Parry
  * @since 3.2
  */
-public class ReactiveElasticsearchTemplate implements ReactiveElasticsearchOperations {
+public class ReactiveElasticsearchTemplate implements ReactiveElasticsearchOperations, ApplicationContextAware {
 
 	private static final Logger QUERY_LOGGER = LoggerFactory
 			.getLogger("org.springframework.data.elasticsearch.core.QUERY");
@@ -108,6 +107,8 @@ public class ReactiveElasticsearchTemplate implements ReactiveElasticsearchOpera
 	private @Nullable RefreshPolicy refreshPolicy = RefreshPolicy.IMMEDIATE;
 	private @Nullable IndicesOptions indicesOptions = IndicesOptions.strictExpandOpenAndForbidClosedIgnoreThrottled();
 
+	private @Nullable ReactiveEntityCallbacks entityCallbacks;
+
 	// region Initialization
 	public ReactiveElasticsearchTemplate(ReactiveElasticsearchClient client) {
 		this(client, new MappingElasticsearchConverter(new SimpleElasticsearchMappingContext()));
@@ -124,7 +125,42 @@ public class ReactiveElasticsearchTemplate implements ReactiveElasticsearchOpera
 		this.exceptionTranslator = new ElasticsearchExceptionTranslator();
 		this.operations = new EntityOperations(this.mappingContext);
 		this.requestFactory = new RequestFactory(converter);
+
+		logVersions();
 	}
+
+	private void logVersions() {
+		getClusterVersion() //
+				.doOnSuccess(VersionInfo::logVersions) //
+				.doOnError(e -> VersionInfo.logVersions(null)) //
+				.subscribe();
+	}
+
+	@Override
+	public void setApplicationContext(ApplicationContext applicationContext) throws BeansException {
+
+		if (entityCallbacks == null) {
+			setEntityCallbacks(ReactiveEntityCallbacks.create(applicationContext));
+		}
+	}
+
+	/**
+	 * Set the {@link ReactiveEntityCallbacks} instance to use when invoking {@link ReactiveEntityCallbacks callbacks}
+	 * like the {@link ReactiveBeforeConvertCallback}.
+	 * <p />
+	 * Overrides potentially existing {@link ReactiveEntityCallbacks}.
+	 *
+	 * @param entityCallbacks must not be {@literal null}.
+	 * @throws IllegalArgumentException if the given instance is {@literal null}.
+	 * @since 4.0
+	 */
+	public void setEntityCallbacks(ReactiveEntityCallbacks entityCallbacks) {
+
+		Assert.notNull(entityCallbacks, "EntityCallbacks must not be null!");
+
+		this.entityCallbacks = entityCallbacks;
+	}
+
 	// endregion
 
 	// region DocumentOperations
@@ -137,12 +173,15 @@ public class ReactiveElasticsearchTemplate implements ReactiveElasticsearchOpera
 
 		Assert.notNull(entity, "Entity must not be null!");
 
-		AdaptibleEntity<T> adaptableEntity = operations.forEntity(entity, converter.getConversionService());
-
-		return doIndex(entity, adaptableEntity, index) //
+		return maybeCallBeforeConvert(entity, index)
+				.flatMap(entityAfterBeforeConversionCallback -> doIndex(entityAfterBeforeConversionCallback, index)) //
 				.map(it -> {
-					return adaptableEntity.populateIdIfNecessary(it.getId());
-				});
+					T savedEntity = it.getT1();
+					IndexResponse indexResponse = it.getT2();
+					AdaptibleEntity<T> adaptableEntity = operations.forEntity(savedEntity, converter.getConversionService());
+					// noinspection ReactiveStreamsNullableInLambdaInTransform
+					return adaptableEntity.populateIdIfNecessary(indexResponse.getId());
+				}).flatMap(saved -> maybeCallAfterSave(saved, index));
 	}
 
 	@Override
@@ -151,30 +190,27 @@ public class ReactiveElasticsearchTemplate implements ReactiveElasticsearchOpera
 	}
 
 	@Override
-	public <T> Flux<T> saveAll(Mono<? extends Collection<? extends T>> entities, IndexCoordinates index) {
+	public <T> Flux<T> saveAll(Mono<? extends Collection<? extends T>> entitiesPublisher, IndexCoordinates index) {
 
-		Assert.notNull(entities, "Entities must not be null!");
+		Assert.notNull(entitiesPublisher, "Entities must not be null!");
 
-		return entities.flatMapMany(entityList -> {
-
-			List<AdaptibleEntity<? extends T>> adaptibleEntities = entityList.stream() //
-					.map(e -> operations.forEntity(e, converter.getConversionService())) //
-					.collect(Collectors.toList());
-
-			if (adaptibleEntities.isEmpty()) {
+		return entitiesPublisher.flatMapMany(entities -> {
+			return Flux.fromIterable(entities) //
+					.concatMap(entity -> maybeCallBeforeConvert(entity, index));
+		}).collectList().map(Entities::new).flatMapMany(entities -> {
+			if (entities.isEmpty()) {
 				return Flux.empty();
 			}
 
-			Iterator<AdaptibleEntity<? extends T>> iterator = adaptibleEntities.iterator();
-			List<IndexQuery> indexRequests = adaptibleEntities.stream() //
-					.map(e -> getIndexQuery(e.getBean(), e)) //
-					.collect(Collectors.toList());
-			return doBulkOperation(indexRequests, BulkOptions.defaultOptions(), index) //
-					.map(bulkItemResponse -> {
+			return doBulkOperation(entities.indexQueries(), BulkOptions.defaultOptions(), index) //
+					.index().flatMap(indexAndResponse -> {
+						T savedEntity = entities.entityAt(indexAndResponse.getT1());
+						BulkItemResponse bulkItemResponse = indexAndResponse.getT2();
 
-						AdaptibleEntity<? extends T> mappedEntity = iterator.next();
-						mappedEntity.populateIdIfNecessary(bulkItemResponse.getResponse().getId());
-						return mappedEntity.getBean();
+						AdaptibleEntity<T> adaptibleEntity = operations.forEntity(savedEntity, converter.getConversionService());
+						adaptibleEntity.populateIdIfNecessary(bulkItemResponse.getResponse().getId());
+
+						return maybeCallAfterSave(savedEntity, index);
 					});
 		});
 	}
@@ -187,16 +223,11 @@ public class ReactiveElasticsearchTemplate implements ReactiveElasticsearchOpera
 		Assert.notNull(query, "Query must not be null");
 		Assert.notEmpty(query.getIds(), "No Id define for Query");
 
+		DocumentCallback<T> callback = new ReadDocumentCallback<>(converter, clazz, index);
+
 		MultiGetRequest request = requestFactory.multiGetRequest(query, index);
 		return Flux.from(execute(client -> client.multiGet(request))) //
-				.handle((result, sink) -> {
-
-					Document document = DocumentAdapters.from(result);
-					T entity = converter.mapDocument(document, clazz);
-					if (entity != null) {
-						sink.next(entity);
-					}
-				});
+				.concatMap(result -> callback.doWith(DocumentAdapters.from(result)));
 	}
 
 	@Override
@@ -223,7 +254,8 @@ public class ReactiveElasticsearchTemplate implements ReactiveElasticsearchOpera
 	protected Flux<BulkItemResponse> doBulkOperation(List<?> queries, BulkOptions bulkOptions, IndexCoordinates index) {
 		BulkRequest bulkRequest = prepareWriteRequest(requestFactory.bulkRequest(queries, bulkOptions, index));
 		return client.bulk(bulkRequest) //
-				.onErrorMap(e -> new ElasticsearchException("Error while bulk for request: " + bulkRequest.toString(), e)) //
+				.onErrorMap(
+						e -> new UncategorizedElasticsearchException("Error while bulk for request: " + bulkRequest.toString(), e)) //
 				.flatMap(this::checkForBulkOperationFailure) //
 				.flatMapMany(response -> Flux.fromArray(response.getItems()));
 	}
@@ -238,7 +270,7 @@ public class ReactiveElasticsearchTemplate implements ReactiveElasticsearchOpera
 					failedDocuments.put(item.getId(), item.getFailureMessage());
 				}
 			}
-			ElasticsearchException exception = new ElasticsearchException(
+			BulkFailureException exception = new BulkFailureException(
 					"Bulk operation has failures. Use ElasticsearchException.getFailedDocuments() for detailed messages ["
 							+ failedDocuments + ']',
 					failedDocuments);
@@ -270,9 +302,8 @@ public class ReactiveElasticsearchTemplate implements ReactiveElasticsearchOpera
 		return doExists(id, index);
 	}
 
-	private Mono<Boolean> doExists(String id, @Nullable IndexCoordinates index) {
-
-		return Mono.defer(() -> doExists(new GetRequest(index.getIndexName(), id)));
+	private Mono<Boolean> doExists(String id, IndexCoordinates index) {
+		return Mono.defer(() -> doExists(requestFactory.getRequest(id, index)));
 	}
 
 	/**
@@ -287,50 +318,46 @@ public class ReactiveElasticsearchTemplate implements ReactiveElasticsearchOpera
 				.onErrorReturn(NoSuchIndexException.class, false);
 	}
 
-	private Mono<IndexResponse> doIndex(Object value, AdaptibleEntity<?> entity, IndexCoordinates index) {
+	private <T> Mono<Tuple2<T, IndexResponse>> doIndex(T entity, IndexCoordinates index) {
 
-		return Mono.defer(() -> {
-			IndexRequest request = getIndexRequest(value, entity, index);
-			request = prepareIndexRequest(value, request);
-			return doIndex(request);
-		});
+		IndexRequest request = requestFactory.indexRequest(getIndexQuery(entity), index);
+		request = prepareIndexRequest(entity, request);
+		return Mono.just(entity).zipWith(doIndex(request));
 	}
 
-	private IndexRequest getIndexRequest(Object value, AdaptibleEntity<?> entity, IndexCoordinates index) {
-		Object id = entity.getId();
+	private IndexQuery getIndexQuery(Object value) {
+		AdaptibleEntity<?> entity = operations.forEntity(value, converter.getConversionService());
 
-		IndexRequest request = id != null ? new IndexRequest(index.getIndexName()).id(converter.convertId(id))
-				: new IndexRequest(index.getIndexName());
-
-		request.source(converter.mapObject(value).toJson(), Requests.INDEX_CONTENT_TYPE);
-
-		if (entity.isVersionedEntity()) {
-
-			Number version = entity.getVersion();
-
-			if (version != null) {
-				request.version(version.longValue());
-				request.versionType(EXTERNAL);
-			}
-		}
-		return request;
-	}
-
-	private IndexQuery getIndexQuery(Object value, AdaptibleEntity<?> entity) {
 		Object id = entity.getId();
 		IndexQuery query = new IndexQuery();
+
 		if (id != null) {
 			query.setId(id.toString());
 		}
 		query.setObject(value);
 
-		if (entity.isVersionedEntity()) {
+		boolean usingSeqNo = false;
+
+		if (entity.hasSeqNoPrimaryTerm()) {
+			SeqNoPrimaryTerm seqNoPrimaryTerm = entity.getSeqNoPrimaryTerm();
+
+			if (seqNoPrimaryTerm != null) {
+				query.setSeqNo(seqNoPrimaryTerm.getSequenceNumber());
+				query.setPrimaryTerm(seqNoPrimaryTerm.getPrimaryTerm());
+				usingSeqNo = true;
+			}
+		}
+
+		// seq_no and version are incompatible in the same request
+		if (!usingSeqNo && entity.isVersionedEntity()) {
+
 			Number version = entity.getVersion();
 
 			if (version != null) {
 				query.setVersion(version.longValue());
 			}
 		}
+
 		return query;
 	}
 
@@ -344,14 +371,14 @@ public class ReactiveElasticsearchTemplate implements ReactiveElasticsearchOpera
 
 		Assert.notNull(id, "Id must not be null!");
 
+		DocumentCallback<T> callback = new ReadDocumentCallback<>(converter, entityType, index);
+
 		return doGet(id, getPersistentEntityFor(entityType), index)
-				.map(it -> converter.mapDocument(DocumentAdapters.from(it), entityType));
+				.flatMap(it -> callback.doWith(DocumentAdapters.from(it)));
 	}
 
 	private Mono<GetResult> doGet(String id, ElasticsearchPersistentEntity<?> entity, IndexCoordinates index) {
-		return Mono.defer(() -> {
-			return doGet(new GetRequest(index.getIndexName(), id));
-		});
+		return Mono.defer(() -> doGet(requestFactory.getRequest(id, index)));
 	}
 
 	/**
@@ -404,8 +431,8 @@ public class ReactiveElasticsearchTemplate implements ReactiveElasticsearchOpera
 	private Mono<String> doDeleteById(String id, IndexCoordinates index) {
 
 		return Mono.defer(() -> {
-
-			return doDelete(prepareDeleteRequest(new DeleteRequest(index.getIndexName(), id)));
+			DeleteRequest request = requestFactory.deleteRequest(id, index);
+			return doDelete(prepareDeleteRequest(request));
 		});
 	}
 
@@ -418,8 +445,7 @@ public class ReactiveElasticsearchTemplate implements ReactiveElasticsearchOpera
 
 		Assert.notNull(query, "Query must not be null!");
 
-		return doDeleteBy(query, getPersistentEntityFor(entityType), index).map(BulkByScrollResponse::getDeleted)
-				.publishNext();
+		return doDeleteBy(query, entityType, index).map(BulkByScrollResponse::getDeleted).publishNext();
 	}
 
 	@Override
@@ -427,13 +453,10 @@ public class ReactiveElasticsearchTemplate implements ReactiveElasticsearchOpera
 		return delete(query, entityType, getIndexCoordinatesFor(entityType));
 	}
 
-	private Flux<BulkByScrollResponse> doDeleteBy(Query query, ElasticsearchPersistentEntity<?> entity,
-			IndexCoordinates index) {
+	private Flux<BulkByScrollResponse> doDeleteBy(Query query, Class<?> entityType, IndexCoordinates index) {
 
 		return Flux.defer(() -> {
-			DeleteByQueryRequest request = new DeleteByQueryRequest(index.getIndexNames());
-			request.setQuery(mappedQuery(query, entity));
-
+			DeleteByQueryRequest request = requestFactory.deleteByQueryRequest(query, entityType, index);
 			return doDeleteBy(prepareDeleteByRequest(request));
 		});
 	}
@@ -491,8 +514,13 @@ public class ReactiveElasticsearchTemplate implements ReactiveElasticsearchOpera
 	 */
 	protected DeleteByQueryRequest prepareDeleteByRequest(DeleteByQueryRequest request) {
 
-		if (refreshPolicy != null && !RefreshPolicy.NONE.equals(refreshPolicy)) {
-			request = request.setRefresh(true);
+		if (refreshPolicy != null) {
+
+			if (RefreshPolicy.NONE.equals(refreshPolicy)) {
+				request = request.setRefresh(false);
+			} else {
+				request = request.setRefresh(true);
+			}
 		}
 
 		if (indicesOptions != null) {
@@ -536,8 +564,8 @@ public class ReactiveElasticsearchTemplate implements ReactiveElasticsearchOpera
 	// region SearchOperations
 	@Override
 	public <T> Flux<SearchHit<T>> search(Query query, Class<?> entityType, Class<T> resultType, IndexCoordinates index) {
-
-		return doFind(query, entityType, index).map(searchDocument -> converter.read(resultType, searchDocument));
+		SearchDocumentCallback<T> callback = new ReadSearchDocumentCallback<>(resultType, index);
+		return doFind(query, entityType, index).concatMap(callback::doWith);
 	}
 
 	@Override
@@ -563,6 +591,24 @@ public class ReactiveElasticsearchTemplate implements ReactiveElasticsearchOpera
 		});
 	}
 
+	@Override
+	public Flux<Aggregation> aggregate(Query query, Class<?> entityType) {
+		return aggregate(query, entityType, getIndexCoordinatesFor(entityType));
+	}
+
+	@Override
+	public Flux<Aggregation> aggregate(Query query, Class<?> entityType, IndexCoordinates index) {
+		return doAggregate(query, entityType, index);
+	}
+
+	private Flux<Aggregation> doAggregate(Query query, Class<?> entityType, IndexCoordinates index) {
+		return Flux.defer(() -> {
+			SearchRequest request = requestFactory.searchRequest(query, entityType, index);
+			request = prepareSearchRequest(request);
+			return doAggregate(request);
+		});
+	}
+
 	@Override
 	public Mono<Long> count(Query query, Class<?> entityType) {
 		return count(query, entityType, getIndexCoordinatesFor(entityType));
@@ -578,49 +624,10 @@ public class ReactiveElasticsearchTemplate implements ReactiveElasticsearchOpera
 
 			SearchRequest request = requestFactory.searchRequest(query, entityType, index);
 			request = prepareSearchRequest(request);
-			request.source().size(0);
-			request.source().trackTotalHits(true);
 			return doCount(request);
 		});
 	}
 
-	private CountRequest buildCountRequest(Query query, ElasticsearchPersistentEntity<?> entity, IndexCoordinates index) {
-
-		CountRequest request = new CountRequest(index.getIndexNames());
-		SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
-		searchSourceBuilder.query(mappedQuery(query, entity));
-		searchSourceBuilder.trackScores(query.getTrackScores());
-
-		QueryBuilder postFilterQuery = mappedFilterQuery(query, entity);
-		if (postFilterQuery != null) {
-			searchSourceBuilder.postFilter(postFilterQuery);
-		}
-
-		if (query.getSourceFilter() != null) {
-			searchSourceBuilder.fetchSource(query.getSourceFilter().getIncludes(), query.getSourceFilter().getExcludes());
-		}
-
-		if (query instanceof NativeSearchQuery && ((NativeSearchQuery) query).getCollapseBuilder() != null) {
-			searchSourceBuilder.collapse(((NativeSearchQuery) query).getCollapseBuilder());
-		}
-
-		sort(query, entity).forEach(searchSourceBuilder::sort);
-
-		if (query.getMinScore() > 0) {
-			searchSourceBuilder.minScore(query.getMinScore());
-		}
-
-		if (query.getIndicesOptions() != null) {
-			request.indicesOptions(query.getIndicesOptions());
-		}
-
-		if (query.getPreference() != null) {
-			request.preference(query.getPreference());
-		}
-		request.source(searchSourceBuilder);
-		return request;
-	}
-
 	/**
 	 * Customization hook on the actual execution result {@link Publisher}. <br />
 	 *
@@ -637,6 +644,22 @@ public class ReactiveElasticsearchTemplate implements ReactiveElasticsearchOpera
 				.onErrorResume(NoSuchIndexException.class, it -> Mono.empty());
 	}
 
+	/**
+	 * Customization hook on the actual execution result {@link Publisher}. <br />
+	 *
+	 * @param request the already prepared {@link SearchRequest} ready to be executed.
+	 * @return a {@link Flux} emitting the result of the operation.
+	 */
+	protected Flux<Aggregation> doAggregate(SearchRequest request) {
+
+		if (QUERY_LOGGER.isDebugEnabled()) {
+			QUERY_LOGGER.debug("Executing doCount: {}", request);
+		}
+
+		return Flux.from(execute(client -> client.aggregate(request))) //
+				.onErrorResume(NoSuchIndexException.class, it -> Flux.empty());
+	}
+
 	/**
 	 * Customization hook on the actual execution result {@link Publisher}. <br />
 	 *
@@ -669,61 +692,6 @@ public class ReactiveElasticsearchTemplate implements ReactiveElasticsearchOpera
 				.map(DocumentAdapters::from).onErrorResume(NoSuchIndexException.class, it -> Mono.empty());
 	}
 
-	@Nullable
-	private QueryBuilder mappedFilterQuery(Query query, ElasticsearchPersistentEntity<?> entity) {
-
-		if (query instanceof NativeSearchQuery) {
-			return ((NativeSearchQuery) query).getFilter();
-		}
-
-		return null;
-	}
-
-	private QueryBuilder mappedQuery(Query query, ElasticsearchPersistentEntity<?> entity) {
-
-		QueryBuilder elasticsearchQuery = null;
-
-		if (query instanceof CriteriaQuery) {
-			converter.updateQuery((CriteriaQuery) query, entity.getType());
-			elasticsearchQuery = new CriteriaQueryProcessor().createQueryFromCriteria(((CriteriaQuery) query).getCriteria());
-		} else if (query instanceof StringQuery) {
-			elasticsearchQuery = new WrapperQueryBuilder(((StringQuery) query).getSource());
-		} else if (query instanceof NativeSearchQuery) {
-			elasticsearchQuery = ((NativeSearchQuery) query).getQuery();
-		} else {
-			throw new IllegalArgumentException(String.format("Unknown query type '%s'.", query.getClass()));
-		}
-
-		return elasticsearchQuery != null ? elasticsearchQuery : QueryBuilders.matchAllQuery();
-	}
-
-	private static List<FieldSortBuilder> sort(Query query, ElasticsearchPersistentEntity<?> entity) {
-
-		if (query.getSort() == null || query.getSort().isUnsorted()) {
-			return Collections.emptyList();
-		}
-
-		List<FieldSortBuilder> mappedSort = new ArrayList<>();
-		for (Sort.Order order : query.getSort()) {
-
-			ElasticsearchPersistentProperty property = entity.getPersistentProperty(order.getProperty());
-			String fieldName = property != null ? property.getFieldName() : order.getProperty();
-
-			FieldSortBuilder sort = SortBuilders.fieldSort(fieldName)
-					.order(order.getDirection().isDescending() ? SortOrder.DESC : SortOrder.ASC);
-
-			if (order.getNullHandling() == Sort.NullHandling.NULLS_FIRST) {
-				sort.missing("_first");
-			} else if (order.getNullHandling() == Sort.NullHandling.NULLS_LAST) {
-				sort.missing("_last");
-			}
-
-			mappedSort.add(sort);
-		}
-
-		return mappedSort;
-	}
-
 	/**
 	 * Customization hook to modify a generated {@link SearchRequest} prior to its execution. Eg. by setting the
 	 * {@link SearchRequest#indicesOptions(IndicesOptions) indices options} if applicable.
@@ -743,6 +711,15 @@ public class ReactiveElasticsearchTemplate implements ReactiveElasticsearchOpera
 	// endregion
 
 	// region Helper methods
+	protected Mono<String> getClusterVersion() {
+		try {
+			return Mono.from(execute(client -> client.info())).map(mainResponse -> mainResponse.getVersion().toString());
+		} catch (Exception ignored) {}
+		return Mono.empty();
+	}
+
+	// endregion
+
 	// Property Setters / Getters
 
 	/**
@@ -792,15 +769,6 @@ public class ReactiveElasticsearchTemplate implements ReactiveElasticsearchOpera
 		return type != null ? mappingContext.getPersistentEntity(type) : null;
 	}
 
-	private Throwable translateException(Throwable throwable) {
-
-		RuntimeException exception = throwable instanceof RuntimeException ? (RuntimeException) throwable
-				: new RuntimeException(throwable.getMessage(), throwable);
-		RuntimeException potentiallyTranslatedException = exceptionTranslator.translateExceptionIfPossible(exception);
-
-		return potentiallyTranslatedException != null ? potentiallyTranslatedException : throwable;
-	}
-
 	/**
 	 * Obtain the {@link ReactiveElasticsearchClient} to operate upon.
 	 *
@@ -812,4 +780,129 @@ public class ReactiveElasticsearchTemplate implements ReactiveElasticsearchOpera
 
 	// endregion
 
+	/**
+	 * translates an Exception if possible. Exceptions that are no {@link RuntimeException}s are wrapped in a
+	 * RuntimeException
+	 *
+	 * @param throwable the Throwable to map
+	 * @return the potentially translated RuntimeException.
+	 * @since 4.0
+	 */
+	private RuntimeException translateException(Throwable throwable) {
+
+		RuntimeException runtimeException = throwable instanceof RuntimeException ? (RuntimeException) throwable
+				: new RuntimeException(throwable.getMessage(), throwable);
+		RuntimeException potentiallyTranslatedException = exceptionTranslator
+				.translateExceptionIfPossible(runtimeException);
+
+		return potentiallyTranslatedException != null ? potentiallyTranslatedException : runtimeException;
+	}
+
+	// region callbacks
+	protected <T> Mono<T> maybeCallBeforeConvert(T entity, IndexCoordinates index) {
+
+		if (null != entityCallbacks) {
+			return entityCallbacks.callback(ReactiveBeforeConvertCallback.class, entity, index);
+		}
+
+		return Mono.just(entity);
+	}
+
+	protected <T> Mono<T> maybeCallAfterSave(T entity, IndexCoordinates index) {
+
+		if (null != entityCallbacks) {
+			return entityCallbacks.callback(ReactiveAfterSaveCallback.class, entity, index);
+		}
+
+		return Mono.just(entity);
+	}
+
+	protected <T> Mono<T> maybeCallAfterConvert(T entity, Document document, IndexCoordinates index) {
+
+		if (null != entityCallbacks) {
+			return entityCallbacks.callback(ReactiveAfterConvertCallback.class, entity, document, index);
+		}
+
+		return Mono.just(entity);
+	}
+	// endregion
+
+	protected interface DocumentCallback<T> {
+
+		@NonNull
+		Mono<T> doWith(@Nullable Document document);
+	}
+
+	protected class ReadDocumentCallback<T> implements DocumentCallback<T> {
+		private final EntityReader<? super T, Document> reader;
+		private final Class<T> type;
+		private final IndexCoordinates index;
+
+		public ReadDocumentCallback(EntityReader<? super T, Document> reader, Class<T> type, IndexCoordinates index) {
+			Assert.notNull(reader, "reader is null");
+			Assert.notNull(type, "type is null");
+
+			this.reader = reader;
+			this.type = type;
+			this.index = index;
+		}
+
+		@NonNull
+		public Mono<T> doWith(@Nullable Document document) {
+			if (document == null) {
+				return Mono.empty();
+			}
+
+			T entity = reader.read(type, document);
+			return maybeCallAfterConvert(entity, document, index);
+		}
+	}
+
+	protected interface SearchDocumentCallback<T> {
+
+		@NonNull
+		Mono<SearchHit<T>> doWith(@NonNull SearchDocument response);
+	}
+
+	protected class ReadSearchDocumentCallback<T> implements SearchDocumentCallback<T> {
+		private final DocumentCallback<T> delegate;
+		private final Class<T> type;
+
+		public ReadSearchDocumentCallback(Class<T> type, IndexCoordinates index) {
+			Assert.notNull(type, "type is null");
+
+			this.delegate = new ReadDocumentCallback<>(converter, type, index);
+			this.type = type;
+		}
+
+		@Override
+		public Mono<SearchHit<T>> doWith(SearchDocument response) {
+			return delegate.doWith(response)
+					.map(entity -> SearchHitMapping.mappingFor(type, converter.getMappingContext()).mapHit(response, entity));
+		}
+	}
+
+	private class Entities<T> {
+		private final List<T> entities;
+
+		private Entities(List<T> entities) {
+			Assert.notNull(entities, "entities cannot be null");
+
+			this.entities = entities;
+		}
+
+		private boolean isEmpty() {
+			return entities.isEmpty();
+		}
+
+		private List<IndexQuery> indexQueries() {
+			return entities.stream().map(ReactiveElasticsearchTemplate.this::getIndexQuery).collect(Collectors.toList());
+		}
+
+		private T entityAt(long index) {
+			// it's safe to cast to int because the original indexed colleciton was fitting in memory
+			int intIndex = (int) index;
+			return entities.get(intIndex);
+		}
+	}
 }

src/main/java/org/springframework/data/elasticsearch/core/ReactiveSearchOperations.java

@@ -19,6 +19,7 @@ import reactor.core.publisher.Flux;
 import reactor.core.publisher.Mono;
 
 import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.search.aggregations.Aggregation;
 import org.springframework.data.domain.Pageable;
 import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates;
 import org.springframework.data.elasticsearch.core.query.Query;
@@ -30,6 +31,7 @@ import org.springframework.data.elasticsearch.core.query.StringQuery;
  * APIs</a>.
  *
  * @author Peter-Josef Meisch
+ * @author Russell Parry
  * @since 4.0
  */
 public interface ReactiveSearchOperations {
@@ -183,4 +185,25 @@ public interface ReactiveSearchOperations {
 	default <T> Flux<SearchHit<T>> search(Query query, Class<T> entityType, IndexCoordinates index) {
 		return search(query, entityType, entityType, index);
 	}
+
+	/**
+	 * Perform an aggregation specified by the given {@link Query query}. <br />
+	 *
+	 * @param query must not be {@literal null}.
+	 * @param entityType must not be {@literal null}.
+	 * @return a {@link Flux} emitting matching aggregations one by one.
+	 * @since 4.0
+	 */
+	Flux<Aggregation> aggregate(Query query, Class<?> entityType);
+
+	/**
+	 * Perform an aggregation specified by the given {@link Query query}. <br />
+	 *
+	 * @param query must not be {@literal null}.
+	 * @param entityType must not be {@literal null}.
+	 * @param index the target index, must not be {@literal null}
+	 * @return a {@link Flux} emitting matching aggregations one by one.
+	 * @since 4.0
+	 */
+	Flux<Aggregation> aggregate(Query query, Class<?> entityType, IndexCoordinates index);
 }

src/main/java/org/springframework/data/elasticsearch/core/RequestFactory.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2019-2020 the original author or authors.
+ * Copyright 2019-2021 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -19,6 +19,7 @@ import static org.elasticsearch.index.query.QueryBuilders.*;
 import static org.springframework.util.CollectionUtils.*;
 
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -28,6 +29,7 @@ import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder;
 import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequestBuilder;
 import org.elasticsearch.action.bulk.BulkRequest;
 import org.elasticsearch.action.bulk.BulkRequestBuilder;
+import org.elasticsearch.action.delete.DeleteRequest;
 import org.elasticsearch.action.get.GetRequest;
 import org.elasticsearch.action.get.GetRequestBuilder;
 import org.elasticsearch.action.get.MultiGetRequest;
@@ -36,6 +38,7 @@ import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.search.SearchRequestBuilder;
+import org.elasticsearch.action.support.ActiveShardCount;
 import org.elasticsearch.action.update.UpdateRequest;
 import org.elasticsearch.action.update.UpdateRequestBuilder;
 import org.elasticsearch.client.Client;
@@ -56,6 +59,7 @@ import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.search.aggregations.AbstractAggregationBuilder;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
 import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
 import org.elasticsearch.search.sort.FieldSortBuilder;
 import org.elasticsearch.search.sort.GeoDistanceSortBuilder;
@@ -81,9 +85,14 @@ import org.springframework.util.StringUtils;
  *
  * @author Peter-Josef Meisch
  * @author Sascha Woo
+ * @author Roman Puchkovskiy
  * @since 4.0
  */
 class RequestFactory {
+
+	// the default max result window size of Elasticsearch
+	static final Integer INDEX_MAX_RESULT_WINDOW = 10_000;
+
 	private final ElasticsearchConverter elasticsearchConverter;
 
 	public RequestFactory(ElasticsearchConverter elasticsearchConverter) {
@@ -100,13 +109,20 @@ class RequestFactory {
 			aliasAction.filter(query.getFilterBuilder());
 		} else if (query.getFilter() != null) {
 			aliasAction.filter(query.getFilter());
-		} else if (!StringUtils.isEmpty(query.getRouting())) {
+		}
+
+		if (!StringUtils.isEmpty(query.getRouting())) {
 			aliasAction.routing(query.getRouting());
-		} else if (!StringUtils.isEmpty(query.getSearchRouting())) {
+		}
+
+		if (!StringUtils.isEmpty(query.getSearchRouting())) {
 			aliasAction.searchRouting(query.getSearchRouting());
-		} else if (!StringUtils.isEmpty(query.getIndexRouting())) {
+		}
+
+		if (!StringUtils.isEmpty(query.getIndexRouting())) {
 			aliasAction.indexRouting(query.getIndexRouting());
 		}
+
 		return aliasAction;
 	}
 
@@ -235,6 +251,10 @@ class RequestFactory {
 		return deleteByQueryRequest;
 	}
 
+	public DeleteRequest deleteRequest(String id, IndexCoordinates index) {
+		return new DeleteRequest(index.getIndexName(), id);
+	}
+
 	@Deprecated
 	public DeleteByQueryRequestBuilder deleteByQueryRequestBuilder(Client client, DeleteQuery deleteQuery,
 			IndexCoordinates index) {
@@ -330,12 +350,21 @@ class RequestFactory {
 			throw new ElasticsearchException(
 					"object or source is null, failed to index the document [id: " + query.getId() + ']');
 		}
+
 		if (query.getVersion() != null) {
 			indexRequest.version(query.getVersion());
 			VersionType versionType = retrieveVersionTypeFromPersistentEntity(query.getObject().getClass());
 			indexRequest.versionType(versionType);
 		}
 
+		if (query.getSeqNo() != null) {
+			indexRequest.setIfSeqNo(query.getSeqNo());
+		}
+
+		if (query.getPrimaryTerm() != null) {
+			indexRequest.setIfPrimaryTerm(query.getPrimaryTerm());
+		}
+
 		return indexRequest;
 	}
 
@@ -368,6 +397,13 @@ class RequestFactory {
 			indexRequestBuilder.setVersionType(versionType);
 		}
 
+		if (query.getSeqNo() != null) {
+			indexRequestBuilder.setIfSeqNo(query.getSeqNo());
+		}
+		if (query.getPrimaryTerm() != null) {
+			indexRequestBuilder.setIfPrimaryTerm(query.getPrimaryTerm());
+		}
+
 		return indexRequestBuilder;
 	}
 
@@ -495,6 +531,38 @@ class RequestFactory {
 			updateRequest.fetchSource(query.getFetchSource());
 		}
 
+		if (query.getFetchSourceIncludes() != null || query.getFetchSourceExcludes() != null) {
+			List<String> includes = query.getFetchSourceIncludes() != null ? query.getFetchSourceIncludes()
+					: Collections.emptyList();
+			List<String> excludes = query.getFetchSourceExcludes() != null ? query.getFetchSourceExcludes()
+					: Collections.emptyList();
+			updateRequest.fetchSource(includes.toArray(new String[0]), excludes.toArray(new String[0]));
+		}
+
+		if (query.getIfSeqNo() != null) {
+			updateRequest.setIfSeqNo(query.getIfSeqNo());
+		}
+
+		if (query.getIfPrimaryTerm() != null) {
+			updateRequest.setIfPrimaryTerm(query.getIfPrimaryTerm());
+		}
+
+		if (query.getRefresh() != null) {
+			updateRequest.setRefreshPolicy(query.getRefresh().name().toLowerCase());
+		}
+
+		if (query.getRetryOnConflict() != null) {
+			updateRequest.retryOnConflict(query.getRetryOnConflict());
+		}
+
+		if (query.getTimeout() != null) {
+			updateRequest.timeout(query.getTimeout());
+		}
+
+		if (query.getWaitForActiveShards() != null) {
+			updateRequest.waitForActiveShards(ActiveShardCount.parseString(query.getWaitForActiveShards()));
+		}
+
 		return updateRequest;
 	}
 
@@ -537,6 +605,38 @@ class RequestFactory {
 			updateRequestBuilder.setFetchSource(query.getFetchSource());
 		}
 
+		if (query.getFetchSourceIncludes() != null || query.getFetchSourceExcludes() != null) {
+			List<String> includes = query.getFetchSourceIncludes() != null ? query.getFetchSourceIncludes()
+					: Collections.emptyList();
+			List<String> excludes = query.getFetchSourceExcludes() != null ? query.getFetchSourceExcludes()
+					: Collections.emptyList();
+			updateRequestBuilder.setFetchSource(includes.toArray(new String[0]), excludes.toArray(new String[0]));
+		}
+
+		if (query.getIfSeqNo() != null) {
+			updateRequestBuilder.setIfSeqNo(query.getIfSeqNo());
+		}
+
+		if (query.getIfPrimaryTerm() != null) {
+			updateRequestBuilder.setIfPrimaryTerm(query.getIfPrimaryTerm());
+		}
+
+		if (query.getRefresh() != null) {
+			updateRequestBuilder.setRefreshPolicy(query.getRefresh().name().toLowerCase());
+		}
+
+		if (query.getRetryOnConflict() != null) {
+			updateRequestBuilder.setRetryOnConflict(query.getRetryOnConflict());
+		}
+
+		if (query.getTimeout() != null) {
+			updateRequestBuilder.setTimeout(query.getTimeout());
+		}
+
+		if (query.getWaitForActiveShards() != null) {
+			updateRequestBuilder.setWaitForActiveShards(ActiveShardCount.parseString(query.getWaitForActiveShards()));
+		}
+
 		return updateRequestBuilder;
 	}
 
@@ -548,6 +648,9 @@ class RequestFactory {
 		SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
 		sourceBuilder.version(true);
 		sourceBuilder.trackScores(query.getTrackScores());
+		if (hasSeqNoPrimaryTermProperty(clazz)) {
+			sourceBuilder.seqNoAndPrimaryTerm(true);
+		}
 
 		if (query.getSourceFilter() != null) {
 			SourceFilter sourceFilter = query.getSourceFilter();
@@ -557,6 +660,9 @@ class RequestFactory {
 		if (query.getPageable().isPaged()) {
 			sourceBuilder.from((int) query.getPageable().getOffset());
 			sourceBuilder.size(query.getPageable().getPageSize());
+		} else {
+			sourceBuilder.from(0);
+			sourceBuilder.size(INDEX_MAX_RESULT_WINDOW);
 		}
 
 		if (!query.getFields().isEmpty()) {
@@ -608,7 +714,21 @@ class RequestFactory {
 		return request;
 	}
 
-	@SuppressWarnings("unchecked")
+	private boolean hasSeqNoPrimaryTermProperty(@Nullable Class<?> entityClass) {
+
+		if (entityClass == null) {
+			return false;
+		}
+
+		if (!elasticsearchConverter.getMappingContext().hasPersistentEntityFor(entityClass)) {
+			return false;
+		}
+
+		ElasticsearchPersistentEntity<?> entity = elasticsearchConverter.getMappingContext()
+				.getRequiredPersistentEntity(entityClass);
+		return entity.hasSeqNoPrimaryTermProperty();
+	}
+
 	public PutMappingRequest putMappingRequest(IndexCoordinates index, Document mapping) {
 		PutMappingRequest request = new PutMappingRequest(index.getIndexName());
 		request.source(mapping);
@@ -641,13 +761,20 @@ class RequestFactory {
 			searchQuery.addSourceFilter(new FetchSourceFilter(toArray(searchQuery.getFields()), null));
 		}
 
+        FetchSourceContext fetchSourceContext = getFetchSourceContext(searchQuery);
+
 		for (String id : searchQuery.getIds()) {
 			MultiGetRequest.Item item = new MultiGetRequest.Item(index.getIndexName(), id);
 
 			if (searchQuery.getRoute() != null) {
 				item = item.routing(searchQuery.getRoute());
 			}
-			items.add(item);
+
+            if (fetchSourceContext != null) {
+                item.fetchSourceContext(fetchSourceContext);
+            }
+
+            items.add(item);
 		}
 		return items;
 	}
@@ -711,6 +838,9 @@ class RequestFactory {
 				.setSearchType(query.getSearchType()) //
 				.setVersion(true) //
 				.setTrackScores(query.getTrackScores());
+		if (hasSeqNoPrimaryTermProperty(clazz)) {
+			searchRequestBuilder.seqNoAndPrimaryTerm(true);
+		}
 
 		if (query.getSourceFilter() != null) {
 			SourceFilter sourceFilter = query.getSourceFilter();
@@ -720,6 +850,9 @@ class RequestFactory {
 		if (query.getPageable().isPaged()) {
 			searchRequestBuilder.setFrom((int) query.getPageable().getOffset());
 			searchRequestBuilder.setSize(query.getPageable().getPageSize());
+		} else {
+			searchRequestBuilder.setFrom(0);
+			searchRequestBuilder.setSize(INDEX_MAX_RESULT_WINDOW);
 		}
 
 		if (!query.getFields().isEmpty()) {
@@ -922,4 +1055,24 @@ class RequestFactory {
 		return values.toArray(valuesAsArray);
 	}
 
+    private FetchSourceContext getFetchSourceContext(Query searchQuery) {
+        FetchSourceContext fetchSourceContext = null;
+        SourceFilter sourceFilter = searchQuery.getSourceFilter();
+
+        if (!isEmpty(searchQuery.getFields())) {
+            if (sourceFilter == null) {
+                sourceFilter = new FetchSourceFilter(toArray(searchQuery.getFields()), null);
+            } else {
+                ArrayList<String> arrayList = new ArrayList<>();
+                Collections.addAll(arrayList, sourceFilter.getIncludes());
+                sourceFilter = new FetchSourceFilter(toArray(arrayList), null);
+            }
+
+            fetchSourceContext = new FetchSourceContext(true, sourceFilter.getIncludes(), sourceFilter.getExcludes());
+        } else if (sourceFilter != null) {
+            fetchSourceContext = new FetchSourceContext(true, sourceFilter.getIncludes(), sourceFilter.getExcludes());
+        }
+        return fetchSourceContext;
+    }
+
 }

src/main/java/org/springframework/data/elasticsearch/core/ResourceUtil.java

@@ -38,8 +38,8 @@ public abstract class ResourceUtil {
 	/**
 	 * Read a {@link ClassPathResource} into a {@link String}.
 	 *
-	 * @param url
-	 * @return
+	 * @param url url the file url
+	 * @return the contents of the file or null if it could not be read
 	 */
 	@Nullable
 	public static String readFileFromClasspath(String url) {
@@ -48,7 +48,7 @@ public abstract class ResourceUtil {
 		try (InputStream is = classPathResource.getInputStream()) {
 			return StreamUtils.copyToString(is, Charset.defaultCharset());
 		} catch (Exception e) {
-			LOGGER.debug(String.format("Failed to load file from url: %s: %s", url, e.getMessage()));
+			LOGGER.warn(String.format("Failed to load file from url: %s: %s", url, e.getMessage()));
 			return null;
 		}
 	}

src/main/java/org/springframework/data/elasticsearch/core/ScrolledPage.java

@@ -2,14 +2,15 @@
 package org.springframework.data.elasticsearch.core;
 
 import org.springframework.data.domain.Page;
-import org.springframework.lang.Nullable;
 
 /**
  * @author Artur Konczak
  * @author Peter-Josef Meisch
+ * @author Sascha Woo
+ * @deprecated since 4.0, will be removed in a future version.
  */
+@Deprecated
 public interface ScrolledPage<T> extends Page<T> {
 
-    @Nullable
-    String getScrollId();
+	String getScrollId();
 }

src/main/java/org/springframework/data/elasticsearch/core/SearchHitMapping.java

@@ -0,0 +1,125 @@
+/*
+ * Copyright 2020 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.springframework.data.elasticsearch.core;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import org.elasticsearch.search.aggregations.Aggregations;
+import org.springframework.data.elasticsearch.core.document.SearchDocument;
+import org.springframework.data.elasticsearch.core.document.SearchDocumentResponse;
+import org.springframework.data.elasticsearch.core.mapping.ElasticsearchPersistentEntity;
+import org.springframework.data.elasticsearch.core.mapping.ElasticsearchPersistentProperty;
+import org.springframework.data.mapping.context.MappingContext;
+import org.springframework.lang.Nullable;
+import org.springframework.util.Assert;
+
+/**
+ * @author Rizwan Idrees
+ * @author Mohsin Husen
+ * @author Christoph Strobl
+ * @author Peter-Josef Meisch
+ * @author Mark Paluch
+ * @author Roman Puchkovskiy
+ * @since 4.0
+ */
+class SearchHitMapping<T> {
+	private final Class<T> type;
+	private final MappingContext<? extends ElasticsearchPersistentEntity<?>, ElasticsearchPersistentProperty> mappingContext;
+
+	private SearchHitMapping(Class<T> type,
+			MappingContext<? extends ElasticsearchPersistentEntity<?>, ElasticsearchPersistentProperty> context) {
+
+		Assert.notNull(type, "type is null");
+		Assert.notNull(context, "context is null");
+
+		this.type = type;
+		this.mappingContext = context;
+	}
+
+	static <T> SearchHitMapping<T> mappingFor(Class<T> entityClass,
+			MappingContext<? extends ElasticsearchPersistentEntity<?>, ElasticsearchPersistentProperty> context) {
+		return new SearchHitMapping<>(entityClass, context);
+	}
+
+	SearchHit<T> mapHit(SearchDocument searchDocument, T content) {
+
+		Assert.notNull(searchDocument, "searchDocument is null");
+		Assert.notNull(content, "content is null");
+
+		String id = searchDocument.hasId() ? searchDocument.getId() : null;
+		float score = searchDocument.getScore();
+		Object[] sortValues = searchDocument.getSortValues();
+		Map<String, List<String>> highlightFields = getHighlightsAndRemapFieldNames(searchDocument);
+
+		return new SearchHit<>(id, score, sortValues, highlightFields, content);
+	}
+
+	SearchHits<T> mapHits(SearchDocumentResponse searchDocumentResponse, List<T> contents) {
+		return mapHitsFromResponse(searchDocumentResponse, contents);
+	}
+
+	SearchScrollHits<T> mapScrollHits(SearchDocumentResponse searchDocumentResponse, List<T> contents) {
+		return mapHitsFromResponse(searchDocumentResponse, contents);
+	}
+
+	private SearchHitsImpl<T> mapHitsFromResponse(SearchDocumentResponse searchDocumentResponse, List<T> contents) {
+
+		Assert.notNull(searchDocumentResponse, "searchDocumentResponse is null");
+		Assert.notNull(contents, "contents is null");
+
+		Assert.isTrue(searchDocumentResponse.getSearchDocuments().size() == contents.size(),
+				"Count of documents must match the count of entities");
+
+		long totalHits = searchDocumentResponse.getTotalHits();
+		float maxScore = searchDocumentResponse.getMaxScore();
+		String scrollId = searchDocumentResponse.getScrollId();
+
+		List<SearchHit<T>> searchHits = new ArrayList<>();
+		List<SearchDocument> searchDocuments = searchDocumentResponse.getSearchDocuments();
+		for (int i = 0; i < searchDocuments.size(); i++) {
+			SearchDocument document = searchDocuments.get(i);
+			T content = contents.get(i);
+			SearchHit<T> hit = mapHit(document, content);
+			searchHits.add(hit);
+		}
+		Aggregations aggregations = searchDocumentResponse.getAggregations();
+		TotalHitsRelation totalHitsRelation = TotalHitsRelation.valueOf(searchDocumentResponse.getTotalHitsRelation());
+
+		return new SearchHitsImpl<>(totalHits, totalHitsRelation, maxScore, scrollId, searchHits, aggregations);
+	}
+
+	@Nullable
+	private Map<String, List<String>> getHighlightsAndRemapFieldNames(SearchDocument searchDocument) {
+		Map<String, List<String>> highlightFields = searchDocument.getHighlightFields();
+
+		if (highlightFields == null) {
+			return null;
+		}
+
+		ElasticsearchPersistentEntity<?> persistentEntity = mappingContext.getPersistentEntity(type);
+		if (persistentEntity == null) {
+			return highlightFields;
+		}
+
+		return highlightFields.entrySet().stream().collect(Collectors.toMap(entry -> {
+			ElasticsearchPersistentProperty property = persistentEntity.getPersistentPropertyWithFieldName(entry.getKey());
+			return property != null ? property.getName() : entry.getKey();
+		}, Map.Entry::getValue));
+	}
+}

src/main/java/org/springframework/data/elasticsearch/core/SearchHitSupport.java

@@ -25,13 +25,16 @@ import org.springframework.data.domain.PageImpl;
 import org.springframework.data.domain.Pageable;
 import org.springframework.data.elasticsearch.core.aggregation.AggregatedPage;
 import org.springframework.data.elasticsearch.core.aggregation.impl.AggregatedPageImpl;
-import org.springframework.data.elasticsearch.support.ReactiveSupport;
+import org.springframework.data.repository.util.ReactiveWrappers;
+import org.springframework.data.util.CloseableIterator;
 import org.springframework.lang.Nullable;
 
 /**
  * Utility class with helper methods for working with {@link SearchHit}.
  * 
  * @author Peter-Josef Meisch
+ * @author Sascha Woo
+ * @author Roman Puchkovskiy
  * @since 4.0
  */
 public final class SearchHitSupport {
@@ -64,8 +67,8 @@ public final class SearchHitSupport {
 		if (result instanceof AggregatedPage<?>) {
 			AggregatedPage<?> page = (AggregatedPage<?>) result;
 			List<?> list = page.getContent().stream().map(SearchHitSupport::unwrapSearchHits).collect(Collectors.toList());
-			return new AggregatedPageImpl<>(list, null, page.getTotalElements(), page.getAggregations(), page.getScrollId(),
-					page.getMaxScore());
+			return new AggregatedPageImpl<>(list, page.getPageable(), page.getTotalElements(), page.getAggregations(),
+					page.getScrollId(), page.getMaxScore());
 
 		}
 
@@ -78,7 +81,11 @@ public final class SearchHitSupport {
 			return unwrapSearchHits(searchHits.getSearchHits());
 		}
 
-		if (ReactiveSupport.isReactorAvailable()) {
+		if (result instanceof SearchHitsIterator<?>) {
+			return unwrapSearchHitsIterator((SearchHitsIterator<?>) result);
+		}
+
+		if (ReactiveWrappers.isAvailable(ReactiveWrappers.ReactiveLibrary.PROJECT_REACTOR)) {
 
 			if (result instanceof Flux) {
 				Flux<?> flux = (Flux<?>) result;
@@ -89,16 +96,43 @@ public final class SearchHitSupport {
 		return result;
 	}
 
+	private static CloseableIterator<?> unwrapSearchHitsIterator(SearchHitsIterator<?> iterator) {
+
+		return new CloseableIterator<Object>() {
+			@Override
+			public boolean hasNext() {
+				return iterator.hasNext();
+			}
+
+			@Override
+			public Object next() {
+				return unwrapSearchHits(iterator.next());
+			}
+
+			@Override
+			public void close() {
+				iterator.close();
+			}
+		};
+	}
+
 	/**
 	 * Builds an {@link AggregatedPage} with the {@link SearchHit} objects from a {@link SearchHits} object.
 	 * 
 	 * @param searchHits, must not be {@literal null}.
 	 * @param pageable, must not be {@literal null}.
 	 * @return the created Page
+	 * @deprecated since 4.0, will be removed in a future version.
 	 */
+	@Deprecated
 	public static <T> AggregatedPage<SearchHit<T>> page(SearchHits<T> searchHits, Pageable pageable) {
-		return new AggregatedPageImpl<>(searchHits.getSearchHits(), pageable, searchHits.getTotalHits(),
-				searchHits.getAggregations(), searchHits.getScrollId(), searchHits.getMaxScore());
+		return new AggregatedPageImpl<>( //
+				searchHits.getSearchHits(), //
+				pageable, //
+				searchHits.getTotalHits(), //
+				searchHits.getAggregations(), //
+				null, //
+				searchHits.getMaxScore());
 	}
 
 	public static <T> SearchPage<T> searchPageFor(SearchHits<T> searchHits, @Nullable Pageable pageable) {

src/main/java/org/springframework/data/elasticsearch/core/SearchHits.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2019-2020 the original author or authors.
+ * Copyright 2020 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -15,143 +15,74 @@
  */
 package org.springframework.data.elasticsearch.core;
 
-import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
 
 import org.elasticsearch.search.aggregations.Aggregations;
 import org.springframework.data.util.Streamable;
 import org.springframework.lang.Nullable;
-import org.springframework.util.Assert;
-import org.springframework.util.StringUtils;
 
 /**
  * Encapsulates a list of {@link SearchHit}s with additional information from the search.
- *
+ * 
  * @param <T> the result data class.
- * @author Peter-Josef Meisch
+ * @author Sascha Woo
  * @since 4.0
  */
-public class SearchHits<T> implements Streamable<SearchHit<T>> {
-
-	private final long totalHits;
-	private final TotalHitsRelation totalHitsRelation;
-	private final float maxScore;
-	private final String scrollId;
-	private final List<? extends SearchHit<T>> searchHits;
-	private final Aggregations aggregations;
-
-	/**
-	 * @param totalHits the number of total hits for the search
-	 * @param totalHitsRelation the relation {@see TotalHitsRelation}, must not be {@literal null}
-	 * @param maxScore the maximum score
-	 * @param scrollId the scroll id if available
-	 * @param searchHits must not be {@literal null}
-	 * @param aggregations the aggregations if available
-	 */
-	public SearchHits(long totalHits, TotalHitsRelation totalHitsRelation, float maxScore, @Nullable String scrollId,
-			List<? extends SearchHit<T>> searchHits, @Nullable Aggregations aggregations) {
-
-		Assert.notNull(searchHits, "searchHits must not be null");
-
-		this.totalHits = totalHits;
-		this.totalHitsRelation = totalHitsRelation;
-		this.maxScore = maxScore;
-		this.scrollId = scrollId;
-		this.searchHits = searchHits;
-		this.aggregations = aggregations;
-	}
-
-	@SuppressWarnings("unchecked")
-	@Override
-	public Iterator<SearchHit<T>> iterator() {
-		return (Iterator<SearchHit<T>>) searchHits.iterator();
-	}
-
-	// region getter
-	/**
-	 * @return the number of total hits.
-	 */
-	public long getTotalHits() {
-		return totalHits;
-	}
-
-	/**
-	 * @return the relation for the total hits
-	 */
-	public TotalHitsRelation getTotalHitsRelation() {
-		return totalHitsRelation;
-	}
-
-	/**
-	 * @return the maximum score
-	 */
-	public float getMaxScore() {
-		return maxScore;
-	}
-
-	/**
-	 * @return the scroll id
-	 */
-	@Nullable
-	public String getScrollId() {
-		return scrollId;
-	}
-
-	/**
-	 * @return the contained {@link SearchHit}s.
-	 */
-	public List<SearchHit<T>> getSearchHits() {
-		return Collections.unmodifiableList(searchHits);
-	}
-	// endregion
-
-	// region SearchHit access
-	/**
-	 * @param index position in List.
-	 * @return the {@link SearchHit} at position {index}
-	 * @throws IndexOutOfBoundsException on invalid index
-	 */
-	public SearchHit<T> getSearchHit(int index) {
-		return searchHits.get(index);
-	}
-	// endregion
-
-	@Override
-	public String toString() {
-		return "SearchHits{" + //
-				"totalHits=" + totalHits + //
-				", totalHitsRelation=" + totalHitsRelation + //
-				", maxScore=" + maxScore + //
-				", scrollId='" + scrollId + '\'' + //
-				", searchHits={" + searchHits.size() + " elements}" + //
-				", aggregations=" + aggregations + //
-				'}';
-	}
-
-	// region aggregations
-	/**
-	 * @return true if aggregations are available
-	 */
-	public boolean hasAggregations() {
-		return aggregations != null;
-	}
+public interface SearchHits<T> extends Streamable<SearchHit<T>> {
 
 	/**
 	 * @return the aggregations.
 	 */
 	@Nullable
-	public Aggregations getAggregations() {
-		return aggregations;
-	}
-	// endregion
+	Aggregations getAggregations();
 
 	/**
-	 * Enum to represent the relation that Elasticsearch returns for the totalHits value {@see <a href=
-	 * "https://www.elastic.co/guide/en/elasticsearch/reference/7.5/search-request-body.html#request-body-search-track-total-hits">Ekasticsearch
-	 * docs</a>}
+	 * @return the maximum score
 	 */
-	public enum TotalHitsRelation {
-		EQUAL_TO, GREATER_THAN_OR_EQUAL_TO
+	float getMaxScore();
+
+	/**
+	 * @param index position in List.
+	 * @return the {@link SearchHit} at position {index}
+	 * @throws IndexOutOfBoundsException on invalid index
+	 */
+	SearchHit<T> getSearchHit(int index);
+
+	/**
+	 * @return the contained {@link SearchHit}s.
+	 */
+	List<SearchHit<T>> getSearchHits();
+
+	/**
+	 * @return the number of total hits.
+	 */
+	long getTotalHits();
+
+	/**
+	 * @return the relation for the total hits
+	 */
+	TotalHitsRelation getTotalHitsRelation();
+
+	/**
+	 * @return true if aggregations are available
+	 */
+	default boolean hasAggregations() {
+		return getAggregations() != null;
 	}
+
+	/**
+	 * @return whether the {@link SearchHits} has search hits.
+	 */
+	default boolean hasSearchHits() {
+		return !getSearchHits().isEmpty();
+	}
+
+	/**
+	 * @return an iterator for {@link SearchHit}
+	 */
+	default Iterator<SearchHit<T>> iterator() {
+		return getSearchHits().iterator();
+	}
+
 }

src/main/java/org/springframework/data/elasticsearch/core/SearchHitsImpl.java

@@ -0,0 +1,117 @@
+/*
+ * Copyright 2019-2020 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.springframework.data.elasticsearch.core;
+
+import java.util.Collections;
+import java.util.List;
+
+import org.elasticsearch.search.aggregations.Aggregations;
+import org.springframework.lang.Nullable;
+import org.springframework.util.Assert;
+
+/**
+ * Basic implementation of {@link SearchScrollHits} 
+ *
+ * @param <T> the result data class.
+ * @author Peter-Josef Meisch
+ * @author Sascha Woo
+ * @since 4.0
+ */
+public class SearchHitsImpl<T> implements SearchScrollHits<T> {
+
+	private final long totalHits;
+	private final TotalHitsRelation totalHitsRelation;
+	private final float maxScore;
+	private final String scrollId;
+	private final List<? extends SearchHit<T>> searchHits;
+	private final Aggregations aggregations;
+
+	/**
+	 * @param totalHits the number of total hits for the search
+	 * @param totalHitsRelation the relation {@see TotalHitsRelation}, must not be {@literal null}
+	 * @param maxScore the maximum score
+	 * @param scrollId the scroll id if available
+	 * @param searchHits must not be {@literal null}
+	 * @param aggregations the aggregations if available
+	 */
+	public SearchHitsImpl(long totalHits, TotalHitsRelation totalHitsRelation, float maxScore, @Nullable String scrollId,
+			List<? extends SearchHit<T>> searchHits, @Nullable Aggregations aggregations) {
+
+		Assert.notNull(searchHits, "searchHits must not be null");
+
+		this.totalHits = totalHits;
+		this.totalHitsRelation = totalHitsRelation;
+		this.maxScore = maxScore;
+		this.scrollId = scrollId;
+		this.searchHits = searchHits;
+		this.aggregations = aggregations;
+	}
+
+	// region getter
+	@Override
+	public long getTotalHits() {
+		return totalHits;
+	}
+
+	@Override
+	public TotalHitsRelation getTotalHitsRelation() {
+		return totalHitsRelation;
+	}
+
+	@Override
+	public float getMaxScore() {
+		return maxScore;
+	}
+
+	@Override
+	@Nullable
+	public String getScrollId() {
+		return scrollId;
+	}
+
+	@Override
+	public List<SearchHit<T>> getSearchHits() {
+		return Collections.unmodifiableList(searchHits);
+	}
+	// endregion
+
+	// region SearchHit access
+	@Override
+	public SearchHit<T> getSearchHit(int index) {
+		return searchHits.get(index);
+	}
+	// endregion
+
+	@Override
+	public String toString() {
+		return "SearchHits{" + //
+				"totalHits=" + totalHits + //
+				", totalHitsRelation=" + totalHitsRelation + //
+				", maxScore=" + maxScore + //
+				", scrollId='" + scrollId + '\'' + //
+				", searchHits={" + searchHits.size() + " elements}" + //
+				", aggregations=" + aggregations + //
+				'}';
+	}
+
+	// region aggregations
+	@Override
+	@Nullable
+	public Aggregations getAggregations() {
+		return aggregations;
+	}
+	// endregion
+}

src/main/java/org/springframework/data/elasticsearch/core/SearchHitsIterator.java

@@ -0,0 +1,60 @@
+/*
+ * Copyright 2020 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.springframework.data.elasticsearch.core;
+
+import org.elasticsearch.search.aggregations.Aggregations;
+import org.springframework.data.util.CloseableIterator;
+import org.springframework.lang.Nullable;
+
+/**
+ * A {@link SearchHitsIterator} encapsulates {@link SearchHit} results that can be wrapped in a Java 8
+ * {@link java.util.stream.Stream}.
+ *
+ * @author Sascha Woo
+ * @param <T>
+ * @since 4.0
+ */
+public interface SearchHitsIterator<T> extends CloseableIterator<SearchHit<T>> {
+
+	/**
+	 * @return the aggregations.
+	 */
+	@Nullable
+	Aggregations getAggregations();
+
+	/**
+	 * @return the maximum score
+	 */
+	float getMaxScore();
+
+	/**
+	 * @return the number of total hits.
+	 */
+	long getTotalHits();
+
+	/**
+	 * @return the relation for the total hits
+	 */
+	TotalHitsRelation getTotalHitsRelation();
+
+	/**
+	 * @return true if aggregations are available
+	 */
+	default boolean hasAggregations() {
+		return getAggregations() != null;
+	}
+
+}

src/main/java/org/springframework/data/elasticsearch/core/SearchOperations.java

@@ -36,6 +36,7 @@ import org.springframework.lang.Nullable;
  * APIs</a>.
  *
  * @author Peter-Josef Meisch
+ * @author Sascha Woo
  * @since 4.0
  */
 public interface SearchOperations {
@@ -155,8 +156,9 @@ public interface SearchOperations {
 	 * @param query the query to execute
 	 * @param clazz the entity clazz used for property mapping
 	 * @param index the index to run the query against
-	 * @return a {@link CloseableIterator} that wraps an Elasticsearch scroll context that needs to be closed in case of *
-	 *         error.
+	 * @return a {@link CloseableIterator} that wraps an Elasticsearch scroll context that needs to be closed. The
+	 *         try-with-resources construct should be used to ensure that the close method is invoked after the operations
+	 *         are completed.
 	 * @deprecated since 4.0, use {@link #searchForStream(Query, Class, IndexCoordinates)}.
 	 */
 	@Deprecated
@@ -237,7 +239,6 @@ public interface SearchOperations {
 		return (AggregatedPage<T>) SearchHitSupport.unwrapSearchHits(aggregatedPage);
 	}
 
-
 	// endregion
 
 	/**
@@ -340,27 +341,29 @@ public interface SearchOperations {
 	<T> SearchHits<T> search(MoreLikeThisQuery query, Class<T> clazz, IndexCoordinates index);
 
 	/**
-	 * Executes the given {@link Query} against elasticsearch and return result as {@link CloseableIterator}.
+	 * Executes the given {@link Query} against elasticsearch and return result as {@link SearchHitsIterator}.
 	 * <p>
 	 *
 	 * @param <T> element return type
 	 * @param query the query to execute
 	 * @param clazz the entity clazz used for property mapping and index name extraction
-	 * @return a {@link CloseableIterator} that wraps an Elasticsearch scroll context that needs to be closed in case of *
-	 *         error.
+	 * @return a {@link SearchHitsIterator} that wraps an Elasticsearch scroll context that needs to be closed. The
+	 *         try-with-resources construct should be used to ensure that the close method is invoked after the operations
+	 *         are completed.
 	 */
-	<T> CloseableIterator<SearchHit<T>> searchForStream(Query query, Class<T> clazz);
+	<T> SearchHitsIterator<T> searchForStream(Query query, Class<T> clazz);
 
 	/**
-	 * Executes the given {@link Query} against elasticsearch and return result as {@link CloseableIterator}.
+	 * Executes the given {@link Query} against elasticsearch and return result as {@link SearchHitsIterator}.
 	 * <p>
 	 *
 	 * @param <T> element return type
 	 * @param query the query to execute
 	 * @param clazz the entity clazz used for property mapping
 	 * @param index the index to run the query against
-	 * @return a {@link CloseableIterator} that wraps an Elasticsearch scroll context that needs to be closed in case of *
-	 *         error.
+	 * @return a {@link SearchHitsIterator} that wraps an Elasticsearch scroll context that needs to be closed. The
+	 *         try-with-resources construct should be used to ensure that the close method is invoked after the operations
+	 *         are completed.
 	 */
-	<T> CloseableIterator<SearchHit<T>> searchForStream(Query query, Class<T> clazz, IndexCoordinates index);
+	<T> SearchHitsIterator<T> searchForStream(Query query, Class<T> clazz, IndexCoordinates index);
 }

src/main/java/org/springframework/data/elasticsearch/core/SearchScrollHits.java

@@ -0,0 +1,34 @@
+/*
+ * Copyright 2020 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.springframework.data.elasticsearch.core;
+
+/**
+ * This interface is used to expose the current {@code scrollId} from the underlying scroll context.
+ * <p>
+ * Internal use only.
+ * 
+ * @author Sascha Woo
+ * @param <T>
+ * @since 4.0
+ */
+public interface SearchScrollHits<T> extends SearchHits<T> {
+
+	/**
+	 * @return the scroll id
+	 */
+	String getScrollId();
+
+}

src/main/java/org/springframework/data/elasticsearch/core/StreamQueries.java

@@ -16,85 +16,112 @@
 package org.springframework.data.elasticsearch.core;
 
 import java.util.Iterator;
+import java.util.List;
 import java.util.NoSuchElementException;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.Consumer;
 import java.util.function.Function;
 
-import org.springframework.data.util.CloseableIterator;
+import org.elasticsearch.search.aggregations.Aggregations;
+import org.springframework.data.elasticsearch.client.util.ScrollState;
+import org.springframework.lang.Nullable;
+import org.springframework.util.Assert;
 
 /**
  * Utility to support streaming queries.
  *
  * @author Mark Paluch
+ * @author Sascha Woo
  * @since 3.2
  */
 abstract class StreamQueries {
 
 	/**
-	 * Stream query results using {@link ScrolledPage}.
+	 * Stream query results using {@link SearchScrollHits}.
 	 *
-	 * @param page the initial page.
-	 * @param continueFunction continuation function accepting the current scrollId.
-	 * @param clearScroll cleanup function accepting the current scrollId.
-	 * @param <T>
-	 * @return the {@link CloseableIterator}.
+	 * @param maxCount the maximum number of entities to return, a value of 0 means that all available entities are
+	 *          returned
+	 * @param searchHits the initial hits
+	 * @param continueScrollFunction function to continue scrolling applies to the current scrollId.
+	 * @param clearScrollConsumer consumer to clear the scroll context by accepting the scrollIds to clear.
+	 * @param <T> the entity type
+	 * @return the {@link SearchHitsIterator}.
 	 */
-	static <T> CloseableIterator<T> streamResults(ScrolledPage<T> page,
-			Function<String, ScrolledPage<T>> continueFunction, Consumer<String> clearScroll) {
+	static <T> SearchHitsIterator<T> streamResults(int maxCount, SearchScrollHits<T> searchHits,
+			Function<String, SearchScrollHits<T>> continueScrollFunction, Consumer<List<String>> clearScrollConsumer) {
 
-		return new CloseableIterator<T>() {
+		Assert.notNull(searchHits, "searchHits must not be null.");
+		Assert.notNull(searchHits.getScrollId(), "scrollId of searchHits must not be null.");
+		Assert.notNull(continueScrollFunction, "continueScrollFunction must not be null.");
+		Assert.notNull(clearScrollConsumer, "clearScrollConsumer must not be null.");
 
-			/** As we couldn't retrieve single result with scroll, store current hits. */
-			private volatile Iterator<T> currentHits = page.iterator();
+		Aggregations aggregations = searchHits.getAggregations();
+		float maxScore = searchHits.getMaxScore();
+		long totalHits = searchHits.getTotalHits();
+		TotalHitsRelation totalHitsRelation = searchHits.getTotalHitsRelation();
 
-			/** The scroll id. */
-			private volatile String scrollId = page.getScrollId();
+		return new SearchHitsIterator<T>() {
 
-			/** If stream is finished (ie: cluster returns no results. */
-			private volatile boolean finished = !currentHits.hasNext();
+			private volatile AtomicInteger currentCount = new AtomicInteger();
+			private volatile Iterator<SearchHit<T>> currentScrollHits = searchHits.iterator();
+			private volatile boolean continueScroll = currentScrollHits.hasNext();
+			private volatile ScrollState scrollState = new ScrollState(searchHits.getScrollId());
 
 			@Override
 			public void close() {
-				try {
-					// Clear scroll on cluster only in case of error (cause elasticsearch auto clear scroll when it's done)
-					if (!finished && scrollId != null && currentHits != null && currentHits.hasNext()) {
-						clearScroll.accept(scrollId);
-					}
-				} finally {
-					currentHits = null;
-					scrollId = null;
-				}
+				clearScrollConsumer.accept(scrollState.getScrollIds());
+			}
+
+			@Override
+			@Nullable
+			public Aggregations getAggregations() {
+				return aggregations;
+			}
+
+			@Override
+			public float getMaxScore() {
+				return maxScore;
+			}
+
+			@Override
+			public long getTotalHits() {
+				return totalHits;
+			}
+
+			@Override
+			public TotalHitsRelation getTotalHitsRelation() {
+				return totalHitsRelation;
 			}
 
 			@Override
 			public boolean hasNext() {
-				// Test if stream is finished
-				if (finished) {
+
+				if (!continueScroll || (maxCount > 0 && currentCount.get() >= maxCount)) {
 					return false;
 				}
-				// Test if it remains hits
-				if (currentHits == null || !currentHits.hasNext()) {
-					// Do a new request
-					ScrolledPage<T> scroll = continueFunction.apply(scrollId);
-					// Save hits and scroll id
-					currentHits = scroll.iterator();
-					finished = !currentHits.hasNext();
-					scrollId = scroll.getScrollId();
+
+				if (!currentScrollHits.hasNext()) {
+					SearchScrollHits<T> nextPage = continueScrollFunction.apply(scrollState.getScrollId());
+					currentScrollHits = nextPage.iterator();
+					scrollState.updateScrollId(nextPage.getScrollId());
+					continueScroll = currentScrollHits.hasNext();
 				}
-				return currentHits.hasNext();
+
+				return currentScrollHits.hasNext();
 			}
 
 			@Override
-			public T next() {
+			public SearchHit<T> next() {
 				if (hasNext()) {
-					return currentHits.next();
+					currentCount.incrementAndGet();
+					return currentScrollHits.next();
 				}
 				throw new NoSuchElementException();
 			}
 
 			@Override
 			public void remove() {
-				throw new UnsupportedOperationException("remove");
+				throw new UnsupportedOperationException();
 			}
 		};
 	}

src/main/java/org/springframework/data/elasticsearch/core/TotalHitsRelation.java

@@ -0,0 +1,30 @@
+/*
+ * Copyright 2020 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.springframework.data.elasticsearch.core;
+
+/**
+ * Enum to represent the relation that Elasticsearch returns for the totalHits value {@see <a href=
+ * "https://www.elastic.co/guide/en/elasticsearch/reference/7.5/search-request-body.html#request-body-search-track-total-hits">Ekasticsearch
+ * docs</a>}
+ * 
+ * @author Peter-Josef Meisch
+ * @author Sascha Woo
+ * @since 4.0
+ */
+public enum TotalHitsRelation {
+	EQUAL_TO, //
+	GREATER_THAN_OR_EQUAL_TO
+}

src/main/java/org/springframework/data/elasticsearch/core/aggregation/impl/AggregatedPageImpl.java

@@ -40,7 +40,7 @@ public class AggregatedPageImpl<T> extends PageImpl<T> implements AggregatedPage
 	@Nullable private String scrollId;
 	private float maxScore;
 
-	private static Pageable pageableOrUnpaged(Pageable pageable) {
+	private static Pageable pageableOrUnpaged(@Nullable Pageable pageable) {
 		return ofNullable(pageable).orElse(Pageable.unpaged());
 	}
 
@@ -82,24 +82,25 @@ public class AggregatedPageImpl<T> extends PageImpl<T> implements AggregatedPage
 		this.maxScore = maxScore;
 	}
 
-	public AggregatedPageImpl(List<T> content, Pageable pageable, long total, Aggregations aggregations) {
+	public AggregatedPageImpl(List<T> content, Pageable pageable, long total, @Nullable Aggregations aggregations) {
 		super(content, pageableOrUnpaged(pageable), total);
 		this.aggregations = aggregations;
 	}
 
-	public AggregatedPageImpl(List<T> content, Pageable pageable, long total, Aggregations aggregations, float maxScore) {
+	public AggregatedPageImpl(List<T> content, Pageable pageable, long total, @Nullable Aggregations aggregations,
+			float maxScore) {
 		this(content, pageableOrUnpaged(pageable), total, aggregations);
 		this.maxScore = maxScore;
 	}
 
-	public AggregatedPageImpl(List<T> content, Pageable pageable, long total, Aggregations aggregations,
+	public AggregatedPageImpl(List<T> content, Pageable pageable, long total, @Nullable Aggregations aggregations,
 			String scrollId) {
 		this(content, pageableOrUnpaged(pageable), total, aggregations);
 		this.scrollId = scrollId;
 	}
 
-	public AggregatedPageImpl(List<T> content, Pageable pageable, long total, Aggregations aggregations, String scrollId,
-			float maxScore) {
+	public AggregatedPageImpl(List<T> content, Pageable pageable, long total, @Nullable Aggregations aggregations,
+			String scrollId, float maxScore) {
 		this(content, pageableOrUnpaged(pageable), total, aggregations, scrollId);
 		this.maxScore = maxScore;
 	}

src/main/java/org/springframework/data/elasticsearch/core/client/support/AliasData.java

@@ -15,15 +15,41 @@
  */
 package org.springframework.data.elasticsearch.core.client.support;
 
-import lombok.Data;
-
-import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
-
-@JsonIgnoreProperties(ignoreUnknown = true)
-@Data
 public class AliasData {
-	String filter = null;
-	String routing = null;
-	String search_routing = null;
-	String index_routing = null;
+	private String filter = null;
+	private String routing = null;
+	private String search_routing = null;
+	private String index_routing = null;
+
+	public String getFilter() {
+		return filter;
+	}
+
+	public void setFilter(String filter) {
+		this.filter = filter;
+	}
+
+	public String getRouting() {
+		return routing;
+	}
+
+	public void setRouting(String routing) {
+		this.routing = routing;
+	}
+
+	public String getSearch_routing() {
+		return search_routing;
+	}
+
+	public void setSearch_routing(String search_routing) {
+		this.search_routing = search_routing;
+	}
+
+	public String getIndex_routing() {
+		return index_routing;
+	}
+
+	public void setIndex_routing(String index_routing) {
+		this.index_routing = index_routing;
+	}
 }

src/main/java/org/springframework/data/elasticsearch/core/convert/ElasticsearchConverter.java

@@ -15,17 +15,8 @@
  */
 package org.springframework.data.elasticsearch.core.convert;
 
-import java.util.List;
-import java.util.stream.Collectors;
-
 import org.springframework.data.convert.EntityConverter;
-import org.springframework.data.domain.Pageable;
-import org.springframework.data.elasticsearch.core.SearchHit;
-import org.springframework.data.elasticsearch.core.SearchHits;
-import org.springframework.data.elasticsearch.core.aggregation.AggregatedPage;
 import org.springframework.data.elasticsearch.core.document.Document;
-import org.springframework.data.elasticsearch.core.document.SearchDocument;
-import org.springframework.data.elasticsearch.core.document.SearchDocumentResponse;
 import org.springframework.data.elasticsearch.core.mapping.ElasticsearchPersistentEntity;
 import org.springframework.data.elasticsearch.core.mapping.ElasticsearchPersistentProperty;
 import org.springframework.data.elasticsearch.core.query.CriteriaQuery;
@@ -39,6 +30,8 @@ import org.springframework.util.Assert;
  * @author Mohsin Husen
  * @author Christoph Strobl
  * @author Peter-Josef Meisch
+ * @author Sasch Woo
+ * @author Roman Puchkovskiy
  */
 public interface ElasticsearchConverter
 		extends EntityConverter<ElasticsearchPersistentEntity<?>, ElasticsearchPersistentProperty, Object, Document> {
@@ -53,59 +46,6 @@ public interface ElasticsearchConverter
 		return new SpelAwareProxyProjectionFactory();
 	}
 
-	// region read
-	/**
-	 * Map a single {@link Document} to an instance of the given type.
-	 *
-	 * @param document the document to map
-	 * @param type must not be {@literal null}.
-	 * @param <T> the class of type
-	 * @return can be {@literal null} if the document is null or {@link Document#isEmpty()} is true.
-	 * @since 4.0
-	 */
-	@Nullable
-	<T> T mapDocument(@Nullable Document document, Class<T> type);
-
-	/**
-	 * Map a list of {@link Document}s to a list of instance of the given type.
-	 *
-	 * @param documents must not be {@literal null}.
-	 * @param type must not be {@literal null}.
-	 * @param <T> the class of type
-	 * @return a list obtained by calling {@link #mapDocument(Document, Class)} on the elements of the list.
-	 * @since 4.0
-	 */
-	default <T> List<T> mapDocuments(List<Document> documents, Class<T> type) {
-		return documents.stream().map(document -> mapDocument(document, type)).collect(Collectors.toList());
-	}
-
-	/**
-	 * builds a {@link SearchHits} from a {@link SearchDocumentResponse}.
-	 * 
-	 * @param <T> the clazz of the type, must not be {@literal null}.
-	 * @param type the type of the returned data, must not be {@literal null}.
-	 * @param searchDocumentResponse the response to read from, must not be {@literal null}.
-	 * @return a SearchHits object
-	 * @since 4.0
-	 */
-	<T> SearchHits<T> read(Class<T> type, SearchDocumentResponse searchDocumentResponse);
-
-	/**
-	 * builds a {@link SearchHit} from a {@link SearchDocument}.
-	 *
-	 * @param <T> the clazz of the type, must not be {@literal null}.
-	 * @param type the type of the returned data, must not be {@literal null}.
-	 * @param searchDocument must not be {@literal null}
-	 * @return SearchHit with all available information filled in
-	 * @since 4.0
-	 */
-	<T> SearchHit<T> read(Class<T> type, SearchDocument searchDocument);
-
-	<T> AggregatedPage<SearchHit<T>> mapResults(SearchDocumentResponse response, Class<T> clazz,
-			@Nullable Pageable pageable);
-
-	// endregion
-
 	// region write
 	/**
 	 * Convert a given {@literal idValue} to its {@link String} representation taking potentially registered

src/main/java/org/springframework/data/elasticsearch/core/convert/ElasticsearchDateConverter.java

@@ -17,10 +17,14 @@ package org.springframework.data.elasticsearch.core.convert;
 
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
+import java.time.Instant;
+import java.time.ZonedDateTime;
 import java.time.temporal.TemporalAccessor;
+import java.util.Date;
 import java.util.concurrent.ConcurrentHashMap;
 
 import org.elasticsearch.common.time.DateFormatter;
+import org.elasticsearch.common.time.DateFormatters;
 import org.springframework.data.elasticsearch.annotations.DateFormat;
 import org.springframework.util.Assert;
 
@@ -73,9 +77,25 @@ final public class ElasticsearchDateConverter {
 	 * @return the formatted object
 	 */
 	public String format(TemporalAccessor accessor) {
+
+		Assert.notNull(accessor, "accessor must not be null");
+
 		return dateFormatter.format(accessor);
 	}
 
+	/**
+	 * Formats the given {@link TemporalAccessor} int a String
+	 *
+	 * @param date must not be {@literal null}
+	 * @return the formatted object
+	 */
+	public String format(Date date) {
+
+		Assert.notNull(date, "accessor must not be null");
+
+		return dateFormatter.format(Instant.ofEpochMilli(date.getTime()));
+	}
+
 	/**
 	 * Parses a String into an object
 	 * 
@@ -85,10 +105,10 @@ final public class ElasticsearchDateConverter {
 	 * @return the new created object
 	 */
 	public <T extends TemporalAccessor> T parse(String input, Class<T> type) {
-		TemporalAccessor accessor = dateFormatter.parse(input);
+		ZonedDateTime zonedDateTime = DateFormatters.from(dateFormatter.parse(input));
 		try {
 			Method method = type.getMethod("from", TemporalAccessor.class);
-			Object o = method.invoke(null, accessor);
+			Object o = method.invoke(null, zonedDateTime);
 			return type.cast(o);
 		} catch (NoSuchMethodException e) {
 			throw new ConversionException("no 'from' factory method found in class " + type.getName());
@@ -96,4 +116,15 @@ final public class ElasticsearchDateConverter {
 			throw new ConversionException("could not create object of class " + type.getName(), e);
 		}
 	}
+
+	/**
+	 * Parses a String into a Date.
+	 *
+	 * @param input the String to parse, must not be {@literal null}.
+	 * @return the new created object
+	 */
+	public Date parse(String input) {
+		ZonedDateTime zonedDateTime = DateFormatters.from(dateFormatter.parse(input));
+		return new Date(Instant.from(zonedDateTime).toEpochMilli());
+	}
 }

src/main/java/org/springframework/data/elasticsearch/core/convert/MappingElasticsearchConverter.java

@@ -15,20 +15,14 @@
  */
 package org.springframework.data.elasticsearch.core.convert;
 
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
+import java.time.temporal.TemporalAccessor;
+import java.util.*;
 import java.util.Map.Entry;
-import java.util.Optional;
-import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.stream.Collectors;
 
-import org.elasticsearch.search.aggregations.Aggregations;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.springframework.beans.BeansException;
 import org.springframework.beans.factory.InitializingBean;
 import org.springframework.context.ApplicationContext;
@@ -38,25 +32,21 @@ import org.springframework.core.convert.ConversionService;
 import org.springframework.core.convert.support.DefaultConversionService;
 import org.springframework.core.convert.support.GenericConversionService;
 import org.springframework.data.convert.CustomConversions;
-import org.springframework.data.convert.EntityInstantiator;
-import org.springframework.data.convert.EntityInstantiators;
-import org.springframework.data.domain.Pageable;
 import org.springframework.data.elasticsearch.ElasticsearchException;
 import org.springframework.data.elasticsearch.annotations.ScriptedField;
-import org.springframework.data.elasticsearch.core.SearchHit;
-import org.springframework.data.elasticsearch.core.SearchHits;
-import org.springframework.data.elasticsearch.core.aggregation.AggregatedPage;
-import org.springframework.data.elasticsearch.core.aggregation.impl.AggregatedPageImpl;
 import org.springframework.data.elasticsearch.core.document.Document;
 import org.springframework.data.elasticsearch.core.document.SearchDocument;
-import org.springframework.data.elasticsearch.core.document.SearchDocumentResponse;
 import org.springframework.data.elasticsearch.core.mapping.ElasticsearchPersistentEntity;
 import org.springframework.data.elasticsearch.core.mapping.ElasticsearchPersistentProperty;
 import org.springframework.data.elasticsearch.core.mapping.ElasticsearchPersistentPropertyConverter;
 import org.springframework.data.elasticsearch.core.query.CriteriaQuery;
+import org.springframework.data.elasticsearch.core.query.Field;
+import org.springframework.data.elasticsearch.core.query.SeqNoPrimaryTerm;
 import org.springframework.data.mapping.PersistentPropertyAccessor;
 import org.springframework.data.mapping.context.MappingContext;
 import org.springframework.data.mapping.model.ConvertingPropertyAccessor;
+import org.springframework.data.mapping.model.EntityInstantiator;
+import org.springframework.data.mapping.model.EntityInstantiators;
 import org.springframework.data.mapping.model.PersistentEntityParameterValueProvider;
 import org.springframework.data.mapping.model.PropertyValueProvider;
 import org.springframework.data.util.ClassTypeInformation;
@@ -77,11 +67,15 @@ import org.springframework.util.ObjectUtils;
  * @author Christoph Strobl
  * @author Peter-Josef Meisch
  * @author Mark Paluch
+ * @author Roman Puchkovskiy
+ * @author Konrad Kurdej
  * @since 3.2
  */
 public class MappingElasticsearchConverter
 		implements ElasticsearchConverter, ApplicationContextAware, InitializingBean {
 
+	private static final Logger LOGGER = LoggerFactory.getLogger(MappingElasticsearchConverter.class);
+
 	private final MappingContext<? extends ElasticsearchPersistentEntity<?>, ElasticsearchPersistentProperty> mappingContext;
 	private final GenericConversionService conversionService;
 
@@ -90,6 +84,8 @@ public class MappingElasticsearchConverter
 
 	private ElasticsearchTypeMapper typeMapper;
 
+	private ConcurrentHashMap<String, Integer> propertyWarnings = new ConcurrentHashMap<>();
+
 	public MappingElasticsearchConverter(
 			MappingContext<? extends ElasticsearchPersistentEntity<?>, ElasticsearchPersistentProperty> mappingContext) {
 		this(mappingContext, null);
@@ -108,6 +104,7 @@ public class MappingElasticsearchConverter
 
 	@Override
 	public void setApplicationContext(ApplicationContext applicationContext) throws BeansException {
+
 		if (mappingContext instanceof ApplicationContextAware) {
 			((ApplicationContextAware) mappingContext).setApplicationContext(applicationContext);
 		}
@@ -145,85 +142,6 @@ public class MappingElasticsearchConverter
 
 	// region read
 
-	@Override
-	public <T> AggregatedPage<SearchHit<T>> mapResults(SearchDocumentResponse response, Class<T> type,
-			@Nullable Pageable pageable) {
-
-		List<SearchHit<T>> results = response.getSearchDocuments().stream() //
-				.map(searchDocument -> read(type, searchDocument)) //
-				.collect(Collectors.toList());
-
-		return new AggregatedPageImpl<>(results, pageable, response);
-	}
-
-	@Override
-	public <T> SearchHits<T> read(Class<T> type, SearchDocumentResponse searchDocumentResponse) {
-
-		Assert.notNull(type, "type must not be null");
-		Assert.notNull(searchDocumentResponse, "searchDocumentResponse must not be null");
-
-		long totalHits = searchDocumentResponse.getTotalHits();
-		float maxScore = searchDocumentResponse.getMaxScore();
-		String scrollId = searchDocumentResponse.getScrollId();
-		List<SearchHit<T>> searchHits = searchDocumentResponse.getSearchDocuments().stream() //
-				.map(searchDocument -> read(type, searchDocument)) //
-				.collect(Collectors.toList());
-		Aggregations aggregations = searchDocumentResponse.getAggregations();
-		SearchHits.TotalHitsRelation totalHitsRelation = SearchHits.TotalHitsRelation
-				.valueOf(searchDocumentResponse.getTotalHitsRelation());
-
-		return new SearchHits<>(totalHits, totalHitsRelation, maxScore, scrollId, searchHits, aggregations);
-	}
-
-	@Override
-	public <T> SearchHit<T> read(Class<T> type, SearchDocument searchDocument) {
-
-		Assert.notNull(type, "type must not be null");
-		Assert.notNull(searchDocument, "searchDocument must not be null");
-
-		String id = searchDocument.hasId() ? searchDocument.getId() : null;
-		float score = searchDocument.getScore();
-		Object[] sortValues = searchDocument.getSortValues();
-		Map<String, List<String>> highlightFields = getHighlightsAndRemapFieldNames(type, searchDocument);
-		T content = mapDocument(searchDocument, type);
-
-		return new SearchHit<T>(id, score, sortValues, highlightFields, content);
-	}
-
-	@Nullable
-	private Map<String, List<String>> getHighlightsAndRemapFieldNames(Class<?> type, SearchDocument searchDocument) {
-		Map<String, List<String>> highlightFields = searchDocument.getHighlightFields();
-
-		if (highlightFields == null) {
-			return null;
-		}
-
-		ElasticsearchPersistentEntity<?> persistentEntity = mappingContext.getPersistentEntity(type);
-		if (persistentEntity == null) {
-			return highlightFields;
-		}
-
-		return highlightFields.entrySet().stream().collect(Collectors.toMap(entry -> {
-			ElasticsearchPersistentProperty property = persistentEntity.getPersistentPropertyWithFieldName(entry.getKey());
-			return property != null ? property.getName() : entry.getKey();
-		}, Entry::getValue));
-	}
-
-	@Override
-	@Nullable
-	public <T> T mapDocument(@Nullable Document document, Class<T> type) {
-
-		if (document == null) {
-			return null;
-		}
-
-		T mappedResult = read(type, document);
-
-		return type.isInterface() || !ClassUtils.isAssignableValue(type, mappedResult)
-				? getProjectionFactory().createProjection(type, mappedResult)
-				: type.cast(mappedResult);
-	}
-
 	@SuppressWarnings("unchecked")
 	@Override
 	public <R> R read(Class<R> type, Document source) {
@@ -289,6 +207,14 @@ public class MappingElasticsearchConverter
 					targetEntity.getPropertyAccessor(result).setProperty(versionProperty, version);
 				}
 			}
+
+			if (targetEntity.hasSeqNoPrimaryTermProperty() && document.hasSeqNo() && document.hasPrimaryTerm()) {
+				if (isAssignedSeqNo(document.getSeqNo()) && isAssignedPrimaryTerm(document.getPrimaryTerm())) {
+					SeqNoPrimaryTerm seqNoPrimaryTerm = new SeqNoPrimaryTerm(document.getSeqNo(), document.getPrimaryTerm());
+					ElasticsearchPersistentProperty property = targetEntity.getRequiredSeqNoPrimaryTermProperty();
+					targetEntity.getPropertyAccessor(result).setProperty(property, seqNoPrimaryTerm);
+				}
+			}
 		}
 
 		if (source instanceof SearchDocument) {
@@ -304,6 +230,14 @@ public class MappingElasticsearchConverter
 
 	}
 
+	private boolean isAssignedSeqNo(long seqNo) {
+		return seqNo >= 0;
+	}
+
+	private boolean isAssignedPrimaryTerm(long primaryTerm) {
+		return primaryTerm > 0;
+	}
+
 	protected <R> R readProperties(ElasticsearchPersistentEntity<?> entity, R instance,
 			ElasticsearchPropertyValueProvider valueProvider) {
 
@@ -312,7 +246,7 @@ public class MappingElasticsearchConverter
 
 		for (ElasticsearchPersistentProperty prop : entity) {
 
-			if (entity.isConstructorArgument(prop) || prop.isScoreProperty()) {
+			if (entity.isConstructorArgument(prop) || prop.isScoreProperty() || !prop.isReadable()) {
 				continue;
 			}
 
@@ -334,11 +268,26 @@ public class MappingElasticsearchConverter
 			return null;
 		}
 
-		if (property.hasPropertyConverter() && String.class.isAssignableFrom(source.getClass())) {
-			source = property.getPropertyConverter().read((String) source);
+		Class<R> rawType = targetType.getType();
+
+		if (property.hasPropertyConverter()) {
+			source = propertyConverterRead(property, source);
+		} else if (TemporalAccessor.class.isAssignableFrom(property.getType())
+				&& !conversions.hasCustomReadTarget(source.getClass(), rawType)) {
+
+			// log at most 5 times
+			String propertyName = property.getOwner().getType().getSimpleName() + '.' + property.getName();
+			String key = propertyName + "-read";
+			int count = propertyWarnings.computeIfAbsent(key, k -> 0);
+			if (count < 5) {
+				LOGGER.warn(
+						"Type {} of property {} is a TemporalAccessor class but has neither a @Field annotation defining the date type nor a registered converter for reading!"
+								+ " It cannot be mapped from a complex object in Elasticsearch!",
+						property.getType().getSimpleName(), propertyName);
+				propertyWarnings.put(key, count + 1);
+			}
 		}
 
-		Class<R> rawType = targetType.getType();
 		if (conversions.hasCustomReadTarget(source.getClass(), rawType)) {
 			return rawType.cast(conversionService.convert(source, rawType));
 		} else if (source instanceof List) {
@@ -350,6 +299,32 @@ public class MappingElasticsearchConverter
 		return (R) readSimpleValue(source, targetType);
 	}
 
+	private Object propertyConverterRead(ElasticsearchPersistentProperty property, Object source) {
+		ElasticsearchPersistentPropertyConverter propertyConverter = Objects
+				.requireNonNull(property.getPropertyConverter());
+
+		if (source instanceof String[]) {
+			// convert to a List
+			source = Arrays.asList((String[]) source);
+		}
+
+		if (source instanceof List) {
+			source = ((List<?>) source).stream().map(it -> convertOnRead(propertyConverter, it)).collect(Collectors.toList());
+		} else if (source instanceof Set) {
+			source = ((Set<?>) source).stream().map(it -> convertOnRead(propertyConverter, it)).collect(Collectors.toSet());
+		} else {
+			source = convertOnRead(propertyConverter, source);
+		}
+		return source;
+	}
+
+	private Object convertOnRead(ElasticsearchPersistentPropertyConverter propertyConverter, Object source) {
+		if (String.class.isAssignableFrom(source.getClass())) {
+			source = propertyConverter.read((String) source);
+		}
+		return source;
+	}
+
 	@SuppressWarnings("unchecked")
 	@Nullable
 	private <R> R readCollectionValue(@Nullable List<?> source, ElasticsearchPersistentProperty property,
@@ -360,16 +335,24 @@ public class MappingElasticsearchConverter
 		}
 
 		Collection<Object> target = createCollectionForValue(targetType, source.size());
+		TypeInformation<?> componentType = targetType.getComponentType();
 
 		for (Object value : source) {
 
-			if (isSimpleType(value)) {
-				target.add(
-						readSimpleValue(value, targetType.getComponentType() != null ? targetType.getComponentType() : targetType));
+			if (value == null) {
+				target.add(null);
+			} else if (componentType != null && !ClassTypeInformation.OBJECT.equals(componentType)
+					&& isSimpleType(componentType.getType())) {
+				target.add(readSimpleValue(value, componentType));
+			} else if (isSimpleType(value)) {
+				target.add(readSimpleValue(value, componentType != null ? componentType : targetType));
 			} else {
 
 				if (value instanceof List) {
 					target.add(readValue(value, property, property.getTypeInformation().getActualType()));
+				} else if (value instanceof Map) {
+					target
+							.add(readMapValue((Map<String, Object>) value, property, property.getTypeInformation().getActualType()));
 				} else {
 					target.add(readEntity(computeGenericValueTypeForRead(property, value), (Map<String, Object>) value));
 				}
@@ -395,27 +378,31 @@ public class MappingElasticsearchConverter
 		Map<String, Object> target = new LinkedHashMap<>();
 		for (Entry<String, Object> entry : source.entrySet()) {
 
-			if (isSimpleType(entry.getValue())) {
-				target.put(entry.getKey(),
-						readSimpleValue(entry.getValue(), targetType.isMap() ? targetType.getComponentType() : targetType));
+			String entryKey = entry.getKey();
+			Object entryValue = entry.getValue();
+
+			if (entryValue == null) {
+				target.put(entryKey, null);
+			} else if (isSimpleType(entryValue)) {
+				target.put(entryKey,
+						readSimpleValue(entryValue, targetType.isMap() ? targetType.getMapValueType() : targetType));
 			} else {
 
-				ElasticsearchPersistentEntity<?> targetEntity = computeGenericValueTypeForRead(property, entry.getValue());
+				ElasticsearchPersistentEntity<?> targetEntity = computeGenericValueTypeForRead(property, entryValue);
 
 				if (targetEntity.getTypeInformation().isMap()) {
 
-					Map<String, Object> valueMap = (Map<String, Object>) entry.getValue();
+					Map<String, Object> valueMap = (Map<String, Object>) entryValue;
 					if (typeMapper.containsTypeInformation(valueMap)) {
-						target.put(entry.getKey(), readEntity(targetEntity, valueMap));
+						target.put(entryKey, readEntity(targetEntity, valueMap));
 					} else {
-						target.put(entry.getKey(), readValue(valueMap, property, targetEntity.getTypeInformation()));
+						target.put(entryKey, readValue(valueMap, property, targetEntity.getTypeInformation()));
 					}
 
 				} else if (targetEntity.getTypeInformation().isCollectionLike()) {
-					target.put(entry.getKey(),
-							readValue(entry.getValue(), property, targetEntity.getTypeInformation().getActualType()));
+					target.put(entryKey, readValue(entryValue, property, targetEntity.getTypeInformation().getActualType()));
 				} else {
-					target.put(entry.getKey(), readEntity(targetEntity, (Map<String, Object>) entry.getValue()));
+					target.put(entryKey, readEntity(targetEntity, (Map<String, Object>) entryValue));
 				}
 			}
 		}
@@ -529,8 +516,21 @@ public class MappingElasticsearchConverter
 			}
 
 			if (property.hasPropertyConverter()) {
-				ElasticsearchPersistentPropertyConverter propertyConverter = property.getPropertyConverter();
-				value = propertyConverter.write(value);
+				value = propertyConverterWrite(property, value);
+			} else if (TemporalAccessor.class.isAssignableFrom(property.getActualType())
+					&& !conversions.hasCustomWriteTarget(value.getClass())) {
+
+				// log at most 5 times
+				String propertyName = entity.getType().getSimpleName() + '.' + property.getName();
+				String key = propertyName + "-write";
+				int count = propertyWarnings.computeIfAbsent(key, k -> 0);
+				if (count < 5) {
+					LOGGER.warn(
+							"Type {} of property {} is a TemporalAccessor class but has neither a @Field annotation defining the date type nor a registered converter for writing!"
+									+ " It will be mapped to a complex object in Elasticsearch!",
+							property.getType().getSimpleName(), propertyName);
+					propertyWarnings.put(key, count + 1);
+				}
 			}
 
 			if (!isSimpleType(value)) {
@@ -544,6 +544,20 @@ public class MappingElasticsearchConverter
 		}
 	}
 
+	private Object propertyConverterWrite(ElasticsearchPersistentProperty property, Object value) {
+		ElasticsearchPersistentPropertyConverter propertyConverter = Objects
+				.requireNonNull(property.getPropertyConverter());
+
+		if (value instanceof List) {
+			value = ((List<?>) value).stream().map(propertyConverter::write).collect(Collectors.toList());
+		} else if (value instanceof Set) {
+			value = ((Set<?>) value).stream().map(propertyConverter::write).collect(Collectors.toSet());
+		} else {
+			value = propertyConverter.write(value);
+		}
+		return value;
+	}
+
 	protected void writeProperty(ElasticsearchPersistentProperty property, Object value, MapValueAccessor sink) {
 
 		Optional<Class<?>> customWriteTarget = conversions.getCustomWriteTarget(value.getClass());
@@ -614,9 +628,18 @@ public class MappingElasticsearchConverter
 		Map<Object, Object> target = new LinkedHashMap<>();
 		Streamable<Entry<String, Object>> mapSource = Streamable.of(value.entrySet());
 
-		if (!typeHint.getActualType().getType().equals(Object.class)
+		TypeInformation<?> actualType = typeHint.getActualType();
+
+		if (actualType != null && !actualType.getType().equals(Object.class)
 				&& isSimpleType(typeHint.getMapValueType().getType())) {
-			mapSource.forEach(it -> target.put(it.getKey(), getWriteSimpleValue(it.getValue())));
+			mapSource.forEach(it -> {
+
+				if (it.getValue() == null) {
+					target.put(it.getKey(), null);
+				} else {
+					target.put(it.getKey(), getWriteSimpleValue(it.getValue()));
+				}
+			});
 		} else {
 
 			mapSource.forEach(it -> {
@@ -646,14 +669,20 @@ public class MappingElasticsearchConverter
 				: Streamable.of(ObjectUtils.toObjectArray(value));
 
 		List<Object> target = new ArrayList<>();
-		if (!typeHint.getActualType().getType().equals(Object.class) && isSimpleType(typeHint.getActualType().getType())) {
-			collectionSource.map(this::getWriteSimpleValue).forEach(target::add);
+		TypeInformation<?> actualType = typeHint.getActualType();
+		Class<?> type = actualType != null ? actualType.getType() : null;
+
+		if (type != null && !type.equals(Object.class) && isSimpleType(type)) {
+			// noinspection ReturnOfNull
+			collectionSource //
+					.map(element -> element != null ? getWriteSimpleValue(element) : null) //
+					.forEach(target::add);
 		} else {
 
 			collectionSource.map(it -> {
 
 				if (it == null) {
-					//noinspection ReturnOfNull
+					// noinspection ReturnOfNull
 					return null;
 				}
 
@@ -721,10 +750,6 @@ public class MappingElasticsearchConverter
 
 	/**
 	 * Compute the type to use by checking the given entity against the store type;
-	 *
-	 * @param entity
-	 * @param source
-	 * @return
 	 */
 	private ElasticsearchPersistentEntity<?> computeClosestEntity(ElasticsearchPersistentEntity<?> entity,
 			Map<String, Object> source) {
@@ -760,11 +785,12 @@ public class MappingElasticsearchConverter
 
 		if (persistentEntity != null) {
 			criteriaQuery.getCriteria().getCriteriaChain().forEach(criteria -> {
-				String name = criteria.getField().getName();
+				Field field = criteria.getField();
+				String name = field.getName();
 				ElasticsearchPersistentProperty property = persistentEntity.getPersistentProperty(name);
 
 				if (property != null && property.getName().equals(name)) {
-					criteria.getField().setName(property.getFieldName());
+					field.setName(property.getFieldName());
 
 					if (property.hasPropertyConverter()) {
 						ElasticsearchPersistentPropertyConverter propertyConverter = property.getPropertyConverter();
@@ -780,6 +806,13 @@ public class MappingElasticsearchConverter
 							}
 						});
 					}
+
+					org.springframework.data.elasticsearch.annotations.Field fieldAnnotation = property.findAnnotation(org.springframework.data.elasticsearch.annotations.Field.class);
+
+					if (fieldAnnotation != null) {
+						field.setFieldType(fieldAnnotation.type());
+					}
+
 				}
 			});
 		}
@@ -797,13 +830,21 @@ public class MappingElasticsearchConverter
 		@Nullable
 		public Object get(ElasticsearchPersistentProperty property) {
 
+			String fieldName = property.getFieldName();
+
 			if (target instanceof Document) {
 				// nested objects may have properties like 'id' which are recognized as isIdProperty() but they are not
 				// Documents
 				Document document = (Document) target;
 
 				if (property.isIdProperty() && document.hasId()) {
-					return document.getId();
+					Object id = null;
+
+					// take the id property from the document source if available
+					if (!fieldName.contains(".")) {
+						id = target.get(fieldName);
+					}
+					return id != null ? id : document.getId();
 				}
 
 				if (property.isVersionProperty() && document.hasVersion()) {
@@ -816,8 +857,6 @@ public class MappingElasticsearchConverter
 				return ((SearchDocument) target).getScore();
 			}
 
-			String fieldName = property.getFieldName();
-
 			if (!fieldName.contains(".")) {
 				return target.get(fieldName);
 			}

src/main/java/org/springframework/data/elasticsearch/core/document/Document.java

@@ -39,6 +39,7 @@ import org.springframework.util.Assert;
  *
  * @author Mark Paluch
  * @author Peter-Josef Meisch
+ * @author Roman Puchkovskiy
  * @since 4.0
  */
 public interface Document extends Map<String, Object> {
@@ -165,6 +166,70 @@ public interface Document extends Map<String, Object> {
 		throw new UnsupportedOperationException();
 	}
 
+	/**
+	 * Return {@literal true} if this {@link Document} is associated with a seq_no.
+	 *
+	 * @return {@literal true} if this {@link Document} is associated with a seq_no, {@literal false} otherwise.
+	 */
+	default boolean hasSeqNo() {
+		return false;
+	}
+
+	/**
+	 * Retrieve the seq_no associated with this {@link Document}.
+	 * <p>
+	 * The default implementation throws {@link UnsupportedOperationException}. It's recommended to check
+	 * {@link #hasSeqNo()} prior to calling this method.
+	 *
+	 * @return the seq_no associated with this {@link Document}.
+	 * @throws IllegalStateException if the underlying implementation supports seq_no's but no seq_no was yet associated
+	 *           with the document.
+	 */
+	default long getSeqNo() {
+		throw new UnsupportedOperationException();
+	}
+
+	/**
+	 * Set the seq_no for this {@link Document}.
+	 * <p>
+	 * The default implementation throws {@link UnsupportedOperationException}.
+	 */
+	default void setSeqNo(long seqNo) {
+		throw new UnsupportedOperationException();
+	}
+
+	/**
+	 * Return {@literal true} if this {@link Document} is associated with a primary_term.
+	 *
+	 * @return {@literal true} if this {@link Document} is associated with a primary_term, {@literal false} otherwise.
+	 */
+	default boolean hasPrimaryTerm() {
+		return false;
+	}
+
+	/**
+	 * Retrieve the primary_term associated with this {@link Document}.
+	 * <p>
+	 * The default implementation throws {@link UnsupportedOperationException}. It's recommended to check
+	 * {@link #hasPrimaryTerm()} prior to calling this method.
+	 *
+	 * @return the primary_term associated with this {@link Document}.
+	 * @throws IllegalStateException if the underlying implementation supports primary_term's but no primary_term was yet
+	 *           associated with the document.
+	 */
+	default long getPrimaryTerm() {
+		throw new UnsupportedOperationException();
+	}
+
+	/**
+	 * Set the primary_term for this {@link Document}.
+	 * <p>
+	 * The default implementation throws {@link UnsupportedOperationException}.
+	 */
+	default void setPrimaryTerm(long primaryTerm) {
+		throw new UnsupportedOperationException();
+	}
+
 	/**
 	 * Returns the value to which the specified {@code key} is mapped, or {@literal null} if this document contains no
 	 * mapping for the key. The value is casted within the method which makes it useful for calling code as it does not

src/main/java/org/springframework/data/elasticsearch/core/document/DocumentAdapters.java

@@ -54,6 +54,7 @@ import com.fasterxml.jackson.core.JsonGenerator;
  *
  * @author Mark Paluch
  * @author Peter-Josef Meisch
+ * @author Roman Puchkovskiy
  * @since 4.0
  */
 public class DocumentAdapters {
@@ -76,12 +77,15 @@ public class DocumentAdapters {
 		}
 
 		if (source.isSourceEmpty()) {
-			return fromDocumentFields(source, source.getId(), source.getVersion());
+			return fromDocumentFields(source, source.getId(), source.getVersion(), source.getSeqNo(),
+					source.getPrimaryTerm());
 		}
 
 		Document document = Document.from(source.getSourceAsMap());
 		document.setId(source.getId());
 		document.setVersion(source.getVersion());
+		document.setSeqNo(source.getSeqNo());
+		document.setPrimaryTerm(source.getPrimaryTerm());
 
 		return document;
 	}
@@ -104,12 +108,15 @@ public class DocumentAdapters {
 		}
 
 		if (source.isSourceEmpty()) {
-			return fromDocumentFields(source, source.getId(), source.getVersion());
+			return fromDocumentFields(source, source.getId(), source.getVersion(), source.getSeqNo(),
+					source.getPrimaryTerm());
 		}
 
 		Document document = Document.from(source.getSource());
 		document.setId(source.getId());
 		document.setVersion(source.getVersion());
+		document.setSeqNo(source.getSeqNo());
+		document.setPrimaryTerm(source.getPrimaryTerm());
 
 		return document;
 	}
@@ -118,16 +125,16 @@ public class DocumentAdapters {
 	 * Creates a List of {@link Document}s from {@link MultiGetResponse}.
 	 * 
 	 * @param source the source {@link MultiGetResponse}, not {@literal null}.
-	 * @return a possibly empty list of the Documents.
+	 * @return a list of Documents, contains null values for not found Documents.
 	 */
 	public static List<Document> from(MultiGetResponse source) {
 
 		Assert.notNull(source, "MultiGetResponse must not be null");
 
-		//noinspection ReturnOfNull
+		// noinspection ReturnOfNull
 		return Arrays.stream(source.getResponses()) //
 				.map(itemResponse -> itemResponse.isFailed() ? null : DocumentAdapters.from(itemResponse.getResponse())) //
-				.filter(Objects::nonNull).collect(Collectors.toList());
+				.collect(Collectors.toList());
 	}
 
 	/**
@@ -150,7 +157,7 @@ public class DocumentAdapters {
 
 		if (sourceRef == null || sourceRef.length() == 0) {
 			return new SearchDocumentAdapter(source.getScore(), source.getSortValues(), source.getFields(), highlightFields,
-					fromDocumentFields(source, source.getId(), source.getVersion()));
+					fromDocumentFields(source, source.getId(), source.getVersion(), source.getSeqNo(), source.getPrimaryTerm()));
 		}
 
 		Document document = Document.from(source.getSourceAsMap());
@@ -159,6 +166,8 @@ public class DocumentAdapters {
 		if (source.getVersion() >= 0) {
 			document.setVersion(source.getVersion());
 		}
+		document.setSeqNo(source.getSeqNo());
+		document.setPrimaryTerm(source.getPrimaryTerm());
 
 		return new SearchDocumentAdapter(source.getScore(), source.getSortValues(), source.getFields(), highlightFields,
 				document);
@@ -170,10 +179,11 @@ public class DocumentAdapters {
 	 * @param documentFields the {@link DocumentField}s backing the {@link Document}.
 	 * @return the adapted {@link Document}.
 	 */
-	public static Document fromDocumentFields(Iterable<DocumentField> documentFields, String id, long version) {
+	public static Document fromDocumentFields(Iterable<DocumentField> documentFields, String id, long version, long seqNo,
+			long primaryTerm) {
 
 		if (documentFields instanceof Collection) {
-			return new DocumentFieldAdapter((Collection<DocumentField>) documentFields, id, version);
+			return new DocumentFieldAdapter((Collection<DocumentField>) documentFields, id, version, seqNo, primaryTerm);
 		}
 
 		List<DocumentField> fields = new ArrayList<>();
@@ -181,7 +191,7 @@ public class DocumentAdapters {
 			fields.add(documentField);
 		}
 
-		return new DocumentFieldAdapter(fields, id, version);
+		return new DocumentFieldAdapter(fields, id, version, seqNo, primaryTerm);
 	}
 
 	// TODO: Performance regarding keys/values/entry-set
@@ -190,11 +200,16 @@ public class DocumentAdapters {
 		private final Collection<DocumentField> documentFields;
 		private final String id;
 		private final long version;
+		private final long seqNo;
+		private final long primaryTerm;
 
-		DocumentFieldAdapter(Collection<DocumentField> documentFields, String id, long version) {
+		DocumentFieldAdapter(Collection<DocumentField> documentFields, String id, long version, long seqNo,
+				long primaryTerm) {
 			this.documentFields = documentFields;
 			this.id = id;
 			this.version = version;
+			this.seqNo = seqNo;
+			this.primaryTerm = primaryTerm;
 		}
 
 		/*
@@ -238,6 +253,52 @@ public class DocumentAdapters {
 			return this.version;
 		}
 
+		/*
+		 * (non-Javadoc)
+		 * @see org.springframework.data.elasticsearch.core.document.Document#hasSeqNo()
+		 */
+		@Override
+		public boolean hasSeqNo() {
+			return true;
+		}
+
+		/*
+		 * (non-Javadoc)
+		 * @see org.springframework.data.elasticsearch.core.document.Document#getSeqNo()
+		 */
+		@Override
+		public long getSeqNo() {
+
+			if (!hasSeqNo()) {
+				throw new IllegalStateException("No seq_no associated with this Document");
+			}
+
+			return this.seqNo;
+		}
+
+		/*
+		 * (non-Javadoc)
+		 * @see org.springframework.data.elasticsearch.core.document.Document#hasPrimaryTerm()
+		 */
+		@Override
+		public boolean hasPrimaryTerm() {
+			return true;
+		}
+
+		/*
+		 * (non-Javadoc)
+		 * @see org.springframework.data.elasticsearch.core.document.Document#getPrimaryTerm()
+		 */
+		@Override
+		public long getPrimaryTerm() {
+
+			if (!hasPrimaryTerm()) {
+				throw new IllegalStateException("No primary_term associated with this Document");
+			}
+
+			return this.primaryTerm;
+		}
+
 		/*
 		 * (non-Javadoc)
 		 * @see java.util.Map#size()
@@ -299,8 +360,7 @@ public class DocumentAdapters {
 		public Object get(Object key) {
 			return documentFields.stream() //
 					.filter(documentField -> documentField.getName().equals(key)) //
-					.map(DocumentField::getValue)
-					.findFirst() //
+					.map(DocumentField::getValue).findFirst() //
 					.orElse(null); //
 
 		}
@@ -558,6 +618,60 @@ public class DocumentAdapters {
 			delegate.setVersion(version);
 		}
 
+		/*
+		 * (non-Javadoc)
+		 * @see org.springframework.data.elasticsearch.core.document.Document#hasSeqNo()
+		 */
+		@Override
+		public boolean hasSeqNo() {
+			return delegate.hasSeqNo();
+		}
+
+		/*
+		 * (non-Javadoc)
+		 * @see org.springframework.data.elasticsearch.core.document.Document#getSeqNo()
+		 */
+		@Override
+		public long getSeqNo() {
+			return delegate.getSeqNo();
+		}
+
+		/*
+		 * (non-Javadoc)
+		 * @see org.springframework.data.elasticsearch.core.document.Document#setSeqNo(long)
+		 */
+		@Override
+		public void setSeqNo(long seqNo) {
+			delegate.setSeqNo(seqNo);
+		}
+
+		/*
+		 * (non-Javadoc)
+		 * @see org.springframework.data.elasticsearch.core.document.Document#hasPrimaryTerm()
+		 */
+		@Override
+		public boolean hasPrimaryTerm() {
+			return delegate.hasPrimaryTerm();
+		}
+
+		/*
+		 * (non-Javadoc)
+		 * @see org.springframework.data.elasticsearch.core.document.Document#getPrimaryTerm()
+		 */
+		@Override
+		public long getPrimaryTerm() {
+			return delegate.getPrimaryTerm();
+		}
+
+		/*
+		 * (non-Javadoc)
+		 * @see org.springframework.data.elasticsearch.core.document.Document#setPrimaryTerm(long)
+		 */
+		@Override
+		public void setPrimaryTerm(long primaryTerm) {
+			delegate.setPrimaryTerm(primaryTerm);
+		}
+
 		/*
 		 * (non-Javadoc)
 		 * @see org.springframework.data.elasticsearch.core.document.Document#get(java.lang.Object, java.lang.Class)

src/main/java/org/springframework/data/elasticsearch/core/document/MapDocument.java

@@ -31,6 +31,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
  * {@link Document} implementation backed by a {@link LinkedHashMap}.
  *
  * @author Mark Paluch
+ * @author Roman Puchkovskiy
  * @since 4.0
  */
 class MapDocument implements Document {
@@ -41,6 +42,8 @@ class MapDocument implements Document {
 
 	private @Nullable String id;
 	private @Nullable Long version;
+	private @Nullable Long seqNo;
+	private @Nullable Long primaryTerm;
 
 	MapDocument() {
 		this(new LinkedHashMap<>());
@@ -114,6 +117,68 @@ class MapDocument implements Document {
 		this.version = version;
 	}
 
+	/*
+	 * (non-Javadoc)
+	 * @see org.springframework.data.elasticsearch.core.document.Document#hasSeqNo()
+	 */
+	@Override
+	public boolean hasSeqNo() {
+		return this.seqNo != null;
+	}
+
+	/*
+	 * (non-Javadoc)
+	 * @see org.springframework.data.elasticsearch.core.document.Document#getSeqNo()
+	 */
+	@Override
+	public long getSeqNo() {
+
+		if (!hasSeqNo()) {
+			throw new IllegalStateException("No seq_no associated with this Document");
+		}
+
+		return this.seqNo;
+	}
+
+	/*
+	 * (non-Javadoc)
+	 * @see org.springframework.data.elasticsearch.core.document.Document#setSeqNo()
+	 */
+	public void setSeqNo(long seqNo) {
+		this.seqNo = seqNo;
+	}
+
+	/*
+	 * (non-Javadoc)
+	 * @see org.springframework.data.elasticsearch.core.document.Document#hasPrimaryTerm()
+	 */
+	@Override
+	public boolean hasPrimaryTerm() {
+		return this.primaryTerm != null;
+	}
+
+	/*
+	 * (non-Javadoc)
+	 * @see org.springframework.data.elasticsearch.core.document.Document#getPrimaryTerm()
+	 */
+	@Override
+	public long getPrimaryTerm() {
+
+		if (!hasPrimaryTerm()) {
+			throw new IllegalStateException("No primary_term associated with this Document");
+		}
+
+		return this.primaryTerm;
+	}
+
+	/*
+	 * (non-Javadoc)
+	 * @see org.springframework.data.elasticsearch.core.document.Document#setPrimaryTerm()
+	 */
+	public void setPrimaryTerm(long primaryTerm) {
+		this.primaryTerm = primaryTerm;
+	}
+
 	/*
 	 * (non-Javadoc)
 	 * @see java.util.Map#size()

src/main/java/org/springframework/data/elasticsearch/core/event/AfterConvertCallback.java

@@ -0,0 +1,42 @@
+/*
+ * Copyright 2020 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.springframework.data.elasticsearch.core.event;
+
+import org.springframework.data.elasticsearch.core.document.Document;
+import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates;
+import org.springframework.data.mapping.callback.EntityCallback;
+
+/**
+ * Callback being invoked after a domain object is materialized from a {@link Document} when reading results.
+ *
+ * @author Roman Puchkovskiy
+ * @since 4.0
+ * @see org.springframework.data.mapping.callback.EntityCallbacks
+ */
+@FunctionalInterface
+public interface AfterConvertCallback<T> extends EntityCallback<T> {
+
+	/**
+	 * Entity callback method invoked after a domain object is materialized from a {@link Document}. Can return either the
+	 * same or a modified instance of the domain object.
+	 *
+	 * @param entity the domain object (the result of the conversion).
+	 * @param document must not be {@literal null}.
+	 * @param indexCoordinates must not be {@literal null}.
+	 * @return the domain object that is the result of reading it from the {@link Document}.
+	 */
+	T onAfterConvert(T entity, Document document, IndexCoordinates indexCoordinates);
+}

src/main/java/org/springframework/data/elasticsearch/core/event/AfterSaveCallback.java

@@ -0,0 +1,41 @@
+/*
+ * Copyright 2020 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.springframework.data.elasticsearch.core.event;
+
+import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates;
+import org.springframework.data.mapping.callback.EntityCallback;
+import org.springframework.data.mapping.callback.EntityCallbacks;
+
+/**
+ * Entity callback triggered after save of an entity.
+ *
+ * @author Roman Puchkovskiy
+ * @since 4.0
+ * @see EntityCallbacks
+ */
+@FunctionalInterface
+public interface AfterSaveCallback<T> extends EntityCallback<T> {
+
+	/**
+	 * Entity callback method invoked after a domain object is saved. Can return either the same or a modified instance of
+	 * the domain object.
+	 *
+	 * @param entity the domain object that was saved.
+	 * @param index must not be {@literal null}.
+	 * @return the domain object that was persisted.
+	 */
+	T onAfterSave(T entity, IndexCoordinates index);
+}

src/main/java/org/springframework/data/elasticsearch/core/event/AuditingEntityCallback.java

@@ -0,0 +1,58 @@
+/*
+ * Copyright 2020 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.springframework.data.elasticsearch.core.event;
+
+import org.springframework.beans.factory.ObjectFactory;
+import org.springframework.core.Ordered;
+import org.springframework.data.auditing.IsNewAwareAuditingHandler;
+import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates;
+import org.springframework.data.mapping.callback.EntityCallback;
+import org.springframework.util.Assert;
+
+/**
+ * {@link EntityCallback} to populate auditing related fields on an entity about to be saved.
+ *
+ * @author Peter-Josef Meisch
+ * @author Roman Puchkovskiy
+ * @since 4.0
+ */
+public class AuditingEntityCallback implements BeforeConvertCallback<Object>, Ordered {
+
+	private final ObjectFactory<IsNewAwareAuditingHandler> auditingHandlerFactory;
+
+	/**
+	 * Creates a new {@link AuditingEntityCallback} using the given {@link IsNewAwareAuditingHandler} provided by the
+	 * given {@link ObjectFactory}.
+	 *
+	 * @param auditingHandlerFactory must not be {@literal null}.
+	 */
+	public AuditingEntityCallback(ObjectFactory<IsNewAwareAuditingHandler> auditingHandlerFactory) {
+
+		Assert.notNull(auditingHandlerFactory, "IsNewAwareAuditingHandler must not be null!");
+
+		this.auditingHandlerFactory = auditingHandlerFactory;
+	}
+
+	@Override
+	public Object onBeforeConvert(Object entity, IndexCoordinates index) {
+		return auditingHandlerFactory.getObject().markAudited(entity);
+	}
+
+	@Override
+	public int getOrder() {
+		return 100;
+	}
+}

src/main/java/org/springframework/data/elasticsearch/core/event/BeforeConvertCallback.java

@@ -0,0 +1,40 @@
+/*
+ * Copyright 2020 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.springframework.data.elasticsearch.core.event;
+
+import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates;
+import org.springframework.data.mapping.callback.EntityCallback;
+
+/**
+ * Callback being invoked before a domain object is converted to be persisted.
+ *
+ * @author Peter-Josef Meisch
+ * @author Roman Puchkovskiy
+ * @since 4.0
+ */
+@FunctionalInterface
+public interface BeforeConvertCallback<T> extends EntityCallback<T> {
+
+	/**
+	 * Callback method that will be invoked before an entity is persisted. Can return the same or a different instance of
+	 * the domain entity class.
+	 * 
+	 * @param entity the entity being converted
+	 * @param index must not be {@literal null}.
+	 * @return the entity to be converted
+	 */
+	T onBeforeConvert(T entity, IndexCoordinates index);
+}

src/main/java/org/springframework/data/elasticsearch/core/event/ReactiveAfterConvertCallback.java

@@ -0,0 +1,43 @@
+/*
+ * Copyright 2020 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.springframework.data.elasticsearch.core.event;
+
+import org.reactivestreams.Publisher;
+import org.springframework.data.elasticsearch.core.document.Document;
+import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates;
+import org.springframework.data.mapping.callback.EntityCallback;
+
+/**
+ * Callback being invoked after a domain object is materialized from a {@link Document} when reading results.
+ *
+ * @author Roman Puchkovskiy
+ * @since 4.0
+ * @see org.springframework.data.mapping.callback.ReactiveEntityCallbacks
+ */
+@FunctionalInterface
+public interface ReactiveAfterConvertCallback<T> extends EntityCallback<T> {
+
+	/**
+	 * Entity callback method invoked after a domain object is materialized from a {@link Document}. Can return either the
+	 * same or a modified instance of the domain object.
+	 *
+	 * @param entity the domain object (the result of the conversion).
+	 * @param document must not be {@literal null}.
+	 * @param indexCoordinates must not be {@literal null}.
+	 * @return a {@link Publisher} emitting the domain object that is the result of reading it from the {@link Document}.
+	 */
+	Publisher<T> onAfterConvert(T entity, Document document, IndexCoordinates indexCoordinates);
+}

src/main/java/org/springframework/data/elasticsearch/core/event/ReactiveAfterSaveCallback.java

@@ -0,0 +1,42 @@
+/*
+ * Copyright 2020 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.springframework.data.elasticsearch.core.event;
+
+import org.reactivestreams.Publisher;
+import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates;
+import org.springframework.data.mapping.callback.EntityCallback;
+import org.springframework.data.mapping.callback.ReactiveEntityCallbacks;
+
+/**
+ * Entity callback triggered after save of an entity.
+ *
+ * @author Roman Puchkovskiy
+ * @since 4.0
+ * @see ReactiveEntityCallbacks
+ */
+@FunctionalInterface
+public interface ReactiveAfterSaveCallback<T> extends EntityCallback<T> {
+
+	/**
+	 * Entity callback method invoked after a domain object is saved. Can return either the same or a modified instance of
+	 * the domain object.
+	 *
+	 * @param entity the domain object that was saved.
+	 * @param index must not be {@literal null}.
+	 * @return a {@link Publisher} emitting the domain object to be returned to the caller.
+	 */
+	Publisher<T> onAfterSave(T entity, IndexCoordinates index);
+}

src/main/java/org/springframework/data/elasticsearch/core/event/ReactiveAuditingEntityCallback.java

@@ -0,0 +1,60 @@
+/*
+ * Copyright 2020 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.springframework.data.elasticsearch.core.event;
+
+import reactor.core.publisher.Mono;
+
+import org.springframework.beans.factory.ObjectFactory;
+import org.springframework.core.Ordered;
+import org.springframework.data.auditing.IsNewAwareAuditingHandler;
+import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates;
+import org.springframework.data.mapping.callback.EntityCallback;
+import org.springframework.util.Assert;
+
+/**
+ * {@link EntityCallback} to populate auditing related fields on an entity about to be saved.
+ *
+ * @author Peter-Josef Meisch
+ * @author Roman Puchkovskiy
+ * @since 4.0
+ */
+public class ReactiveAuditingEntityCallback implements ReactiveBeforeConvertCallback<Object>, Ordered {
+
+	private final ObjectFactory<IsNewAwareAuditingHandler> auditingHandlerFactory;
+
+	/**
+	 * Creates a new {@link ReactiveAuditingEntityCallback} using the given {@link IsNewAwareAuditingHandler} provided by
+	 * the given {@link ObjectFactory}.
+	 *
+	 * @param auditingHandlerFactory must not be {@literal null}.
+	 */
+	public ReactiveAuditingEntityCallback(ObjectFactory<IsNewAwareAuditingHandler> auditingHandlerFactory) {
+
+		Assert.notNull(auditingHandlerFactory, "IsNewAwareAuditingHandler must not be null!");
+
+		this.auditingHandlerFactory = auditingHandlerFactory;
+	}
+
+	@Override
+	public Mono<Object> onBeforeConvert(Object entity, IndexCoordinates index) {
+		return Mono.just(auditingHandlerFactory.getObject().markAudited(entity));
+	}
+
+	@Override
+	public int getOrder() {
+		return 100;
+	}
+}

src/main/java/org/springframework/data/elasticsearch/core/event/ReactiveBeforeConvertCallback.java

@@ -0,0 +1,41 @@
+/*
+ * Copyright 2020 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.springframework.data.elasticsearch.core.event;
+
+import org.reactivestreams.Publisher;
+import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates;
+import org.springframework.data.mapping.callback.EntityCallback;
+
+/**
+ * Callback being invoked before a domain object is converted to be persisted.
+ *
+ * @author Peter-Josef Meisch
+ * @author Roman Puchkovskiy
+ * @since 4.0
+ */
+@FunctionalInterface
+public interface ReactiveBeforeConvertCallback<T> extends EntityCallback<T> {
+
+	/**
+	 * Callback method that will be invoked before an entity is persisted. Can return the same or a different instance of
+	 * the domain entity class.
+	 * 
+	 * @param entity the entity being converted
+	 * @param index must not be {@literal null}.
+	 * @return the entity to be converted
+	 */
+	Publisher<T> onBeforeConvert(T entity, IndexCoordinates index);
+}

src/main/java/org/springframework/data/elasticsearch/core/event/package-info.java

@@ -0,0 +1,6 @@
+/**
+ * classes and interfaces related to Spring Data Elasticsearch events and callbacks.
+ */
+@org.springframework.lang.NonNullApi
+@org.springframework.lang.NonNullFields
+package org.springframework.data.elasticsearch.core.event;

src/main/java/org/springframework/data/elasticsearch/core/geo/GeoPoint.java

@@ -17,6 +17,8 @@ package org.springframework.data.elasticsearch.core.geo;
 
 import org.springframework.data.geo.Point;
 
+import java.util.Objects;
+
 /**
  * geo-location used for #{@link org.springframework.data.elasticsearch.core.query.Criteria}.
  *
@@ -60,6 +62,20 @@ public class GeoPoint {
 		return new Point(point.getLat(), point.getLon());
 	}
 
+	@Override
+	public boolean equals(Object o) {
+		if (this == o) return true;
+		if (o == null || getClass() != o.getClass()) return false;
+		GeoPoint geoPoint = (GeoPoint) o;
+		return Double.compare(geoPoint.lat, lat) == 0 &&
+				Double.compare(geoPoint.lon, lon) == 0;
+	}
+
+	@Override
+	public int hashCode() {
+		return Objects.hash(lat, lon);
+	}
+
 	@Override
 	public String toString() {
 		return "GeoPoint{" +

src/main/java/org/springframework/data/elasticsearch/core/index/MappingBuilder.java

@@ -81,6 +81,7 @@ public class MappingBuilder {
 	private static final String FIELD_PARENT = "_parent";
 	private static final String FIELD_CONTEXT_NAME = "name";
 	private static final String FIELD_CONTEXT_TYPE = "type";
+	private static final String FIELD_CONTEXT_PATH = "path";
 	private static final String FIELD_CONTEXT_PRECISION = "precision";
 	private static final String FIELD_DYNAMIC_TEMPLATES = "dynamic_templates";
 
@@ -136,8 +137,8 @@ public class MappingBuilder {
 		}
 	}
 
-	private void mapEntity(XContentBuilder builder, @Nullable ElasticsearchPersistentEntity entity, boolean isRootObject,
-			String nestedObjectFieldName, boolean nestedOrObjectField, FieldType fieldType,
+	private void mapEntity(XContentBuilder builder, @Nullable ElasticsearchPersistentEntity<?> entity,
+			boolean isRootObject, String nestedObjectFieldName, boolean nestedOrObjectField, FieldType fieldType,
 			@Nullable Field parentFieldAnnotation, @Nullable DynamicMapping dynamicMapping) throws IOException {
 
 		boolean writeNestedProperties = !isRootObject && (isAnyPropertyAnnotatedWithField(entity) || nestedOrObjectField);
@@ -149,7 +150,7 @@ public class MappingBuilder {
 
 			if (nestedOrObjectField && FieldType.Nested == fieldType && parentFieldAnnotation != null
 					&& parentFieldAnnotation.includeInParent()) {
-				builder.field("include_in_parent", parentFieldAnnotation.includeInParent());
+				builder.field("include_in_parent", true);
 			}
 		}
 
@@ -166,6 +167,15 @@ public class MappingBuilder {
 						return;
 					}
 
+					if (property.isSeqNoPrimaryTermProperty()) {
+						if (property.isAnnotationPresent(Field.class)) {
+							logger.warn("Property {} of {} is annotated for inclusion in mapping, but its type is " + //
+							"SeqNoPrimaryTerm that is never mapped, so it is skipped", //
+									property.getFieldName(), entity.getType());
+						}
+						return;
+					}
+
 					buildPropertyMapping(builder, isRootObject, property);
 				} catch (IOException e) {
 					logger.warn("error mapping property with name {}", property.getName(), e);
@@ -197,37 +207,35 @@ public class MappingBuilder {
 			}
 		}
 
-		boolean isGeoPointProperty = isGeoPointProperty(property);
+		if (isGeoPointProperty(property)) {
+			applyGeoPointFieldMapping(builder, property);
+			return;
+		}
+
+		Field fieldAnnotation = property.findAnnotation(Field.class);
 		boolean isCompletionProperty = isCompletionProperty(property);
 		boolean isNestedOrObjectProperty = isNestedOrObjectProperty(property);
 
-		Field fieldAnnotation = property.findAnnotation(Field.class);
-		if (!isGeoPointProperty && !isCompletionProperty && property.isEntity() && hasRelevantAnnotation(property)) {
+		if (!isCompletionProperty && property.isEntity() && hasRelevantAnnotation(property)) {
 
 			if (fieldAnnotation == null) {
 				return;
 			}
 
-			Iterator<? extends TypeInformation<?>> iterator = property.getPersistentEntityTypes().iterator();
-			ElasticsearchPersistentEntity<?> persistentEntity = iterator.hasNext()
-					? elasticsearchConverter.getMappingContext().getPersistentEntity(iterator.next())
-					: null;
-
-			mapEntity(builder, persistentEntity, false, property.getFieldName(), isNestedOrObjectProperty,
-					fieldAnnotation.type(), fieldAnnotation, property.findAnnotation(DynamicMapping.class));
-
 			if (isNestedOrObjectProperty) {
+				Iterator<? extends TypeInformation<?>> iterator = property.getPersistentEntityTypes().iterator();
+				ElasticsearchPersistentEntity<?> persistentEntity = iterator.hasNext()
+						? elasticsearchConverter.getMappingContext().getPersistentEntity(iterator.next())
+						: null;
+
+				mapEntity(builder, persistentEntity, false, property.getFieldName(), isNestedOrObjectProperty,
+						fieldAnnotation.type(), fieldAnnotation, property.findAnnotation(DynamicMapping.class));
 				return;
 			}
 		}
 
 		MultiField multiField = property.findAnnotation(MultiField.class);
 
-		if (isGeoPointProperty) {
-			applyGeoPointFieldMapping(builder, property);
-			return;
-		}
-
 		if (isCompletionProperty) {
 			CompletionField completionField = property.findAnnotation(CompletionField.class);
 			applyCompletionFieldMapping(builder, property, completionField);
@@ -281,9 +289,15 @@ public class MappingBuilder {
 					builder.startObject();
 					builder.field(FIELD_CONTEXT_NAME, context.name());
 					builder.field(FIELD_CONTEXT_TYPE, context.type().name().toLowerCase());
+
 					if (context.precision().length() > 0) {
 						builder.field(FIELD_CONTEXT_PRECISION, context.precision());
 					}
+
+					if (StringUtils.hasText(context.path())) {
+						builder.field(FIELD_CONTEXT_PATH, context.path());
+					}
+
 					builder.endObject();
 				}
 				builder.endArray();
@@ -308,6 +322,15 @@ public class MappingBuilder {
 	private void addSingleFieldMapping(XContentBuilder builder, ElasticsearchPersistentProperty property,
 			Field annotation, boolean nestedOrObjectField) throws IOException {
 
+		// build the property json, if empty skip it as this is no valid mapping
+		XContentBuilder propertyBuilder = jsonBuilder().startObject();
+		addFieldMappingParameters(propertyBuilder, annotation, nestedOrObjectField);
+		propertyBuilder.endObject().close();
+
+		if ("{}".equals(propertyBuilder.getOutputStream().toString())) {
+			return;
+		}
+
 		builder.startObject(property.getFieldName());
 		addFieldMappingParameters(builder, annotation, nestedOrObjectField);
 		builder.endObject();
@@ -343,7 +366,7 @@ public class MappingBuilder {
 		MappingParameters mappingParameters = MappingParameters.from(annotation);
 
 		if (!nestedOrObjectField && mappingParameters.isStore()) {
-			builder.field(FIELD_PARAM_STORE, mappingParameters.isStore());
+			builder.field(FIELD_PARAM_STORE, true);
 		}
 		mappingParameters.writeTypeAndParametersTo(builder);
 	}

src/main/java/org/springframework/data/elasticsearch/core/index/MappingParameters.java

@@ -18,6 +18,7 @@ package org.springframework.data.elasticsearch.core.index;
 import java.io.IOException;
 import java.lang.annotation.Annotation;
 
+import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.springframework.data.elasticsearch.annotations.DateFormat;
 import org.springframework.data.elasticsearch.annotations.Field;
@@ -36,6 +37,7 @@ import org.springframework.util.StringUtils;
  * {@link org.springframework.data.elasticsearch.annotations.InnerField} annotation.
  *
  * @author Peter-Josef Meisch
+ * @author Aleksei Arsenev
  * @since 4.0
  */
 public final class MappingParameters {
@@ -64,6 +66,7 @@ public final class MappingParameters {
 	static final String FIELD_PARAM_SIMILARITY = "similarity";
 	static final String FIELD_PARAM_TERM_VECTOR = "term_vector";
 	static final String FIELD_PARAM_TYPE = "type";
+	static final String FIELD_PARAM_MAX_SHINGLE_SIZE = "max_shingle_size";
 
 	private boolean index = true;
 	private boolean store = false;
@@ -88,6 +91,7 @@ public final class MappingParameters {
 	private Similarity similarity = Similarity.Default;
 	private TermVector termVector = TermVector.none;
 	private double scalingFactor = 1.0;
+	@Nullable private Integer maxShingleSize;
 
 	/**
 	 * extracts the mapping parameters from the relevant annotations.
@@ -136,6 +140,11 @@ public final class MappingParameters {
 		similarity = field.similarity();
 		termVector = field.termVector();
 		scalingFactor = field.scalingFactor();
+		maxShingleSize = field.maxShingleSize() >= 0 ? field.maxShingleSize() : null;
+		Assert.isTrue(type != FieldType.Search_As_You_Type //
+				|| maxShingleSize == null //
+				|| (maxShingleSize >= 2 && maxShingleSize <= 4), //
+				"maxShingleSize must be in inclusive range from 2 to 4 for field type search_as_you_type");
 	}
 
 	private MappingParameters(InnerField field) {
@@ -165,6 +174,11 @@ public final class MappingParameters {
 		similarity = field.similarity();
 		termVector = field.termVector();
 		scalingFactor = field.scalingFactor();
+		maxShingleSize = field.maxShingleSize() >= 0 ? field.maxShingleSize() : null;
+		Assert.isTrue(type != FieldType.Search_As_You_Type //
+				|| maxShingleSize == null //
+				|| (maxShingleSize >= 2 && maxShingleSize <= 4), //
+				"maxShingleSize must be in inclusive range from 2 to 4 for field type search_as_you_type");
 	}
 
 	public boolean isStore() {
@@ -269,5 +283,9 @@ public final class MappingParameters {
 		if (type == FieldType.Scaled_Float) {
 			builder.field(FIELD_PARAM_SCALING_FACTOR, scalingFactor);
 		}
+
+		if (maxShingleSize != null) {
+			builder.field(FIELD_PARAM_MAX_SHINGLE_SIZE, maxShingleSize);
+		}
 	}
 }

src/main/java/org/springframework/data/elasticsearch/core/mapping/ElasticsearchPersistentEntity.java

@@ -17,6 +17,7 @@ package org.springframework.data.elasticsearch.core.mapping;
 
 import org.elasticsearch.index.VersionType;
 import org.springframework.data.elasticsearch.annotations.Field;
+import org.springframework.data.elasticsearch.core.query.SeqNoPrimaryTerm;
 import org.springframework.data.mapping.PersistentEntity;
 import org.springframework.lang.Nullable;
 
@@ -30,6 +31,7 @@ import org.springframework.lang.Nullable;
  * @author Oliver Gierke
  * @author Ivan Greene
  * @author Peter-Josef Meisch
+ * @author Roman Puchkovskiy
  */
 public interface ElasticsearchPersistentEntity<T> extends PersistentEntity<T, ElasticsearchPersistentProperty> {
 
@@ -58,7 +60,8 @@ public interface ElasticsearchPersistentEntity<T> extends PersistentEntity<T, El
 
 	String settingPath();
 
-	@Nullable VersionType getVersionType();
+	@Nullable
+	VersionType getVersionType();
 
 	boolean isCreateIndexAndMapping();
 
@@ -96,4 +99,41 @@ public interface ElasticsearchPersistentEntity<T> extends PersistentEntity<T, El
 	 */
 	@Nullable
 	ElasticsearchPersistentProperty getPersistentPropertyWithFieldName(String fieldName);
+
+	/**
+	 * Returns whether the {@link ElasticsearchPersistentEntity} has a {@link SeqNoPrimaryTerm} property. If this call
+	 * returns {@literal true}, {@link #getSeqNoPrimaryTermProperty()} will return a non-{@literal null} value.
+	 *
+	 * @return false when {@link ElasticsearchPersistentEntity} does not define a SeqNoPrimaryTerm property.
+	 * @since 4.0
+	 */
+	boolean hasSeqNoPrimaryTermProperty();
+
+	/**
+	 * Returns the {@link SeqNoPrimaryTerm} property of the {@link ElasticsearchPersistentEntity}. Can be {@literal null}
+	 * in case no such property is available on the entity.
+	 *
+	 * @return the {@link SeqNoPrimaryTerm} {@link ElasticsearchPersistentProperty} of the {@link PersistentEntity} or
+	 *         {@literal null} if not defined.
+	 * @since 4.0
+	 */
+	@Nullable
+	ElasticsearchPersistentProperty getSeqNoPrimaryTermProperty();
+
+	/**
+	 * Returns the {@link SeqNoPrimaryTerm} property of the {@link ElasticsearchPersistentEntity} or throws an
+	 * IllegalStateException in case no such property is available on the entity.
+	 *
+	 * @return the {@link SeqNoPrimaryTerm} {@link ElasticsearchPersistentProperty} of the {@link PersistentEntity}.
+	 * @since 4.0
+	 */
+	default ElasticsearchPersistentProperty getRequiredSeqNoPrimaryTermProperty() {
+		ElasticsearchPersistentProperty property = this.getSeqNoPrimaryTermProperty();
+		if (property != null) {
+			return property;
+		} else {
+			throw new IllegalStateException(
+					String.format("Required SeqNoPrimaryTerm property not found for %s!", this.getType()));
+		}
+	}
 }

src/main/java/org/springframework/data/elasticsearch/core/mapping/ElasticsearchPersistentProperty.java

@@ -28,6 +28,7 @@ import org.springframework.lang.Nullable;
  * @author Sascha Woo
  * @author Oliver Gierke
  * @author Peter-Josef Meisch
+ * @author Roman Puchkovskiy
  */
 public interface ElasticsearchPersistentProperty extends PersistentProperty<ElasticsearchPersistentProperty> {
 
@@ -42,8 +43,8 @@ public interface ElasticsearchPersistentProperty extends PersistentProperty<Elas
 	 * Returns whether the current property is a <em>potential</em> score property of the owning
 	 * {@link ElasticsearchPersistentEntity}. This method is mainly used by {@link ElasticsearchPersistentEntity}
 	 * implementation to discover score property candidates on {@link ElasticsearchPersistentEntity} creation you should
-	 * rather call {@link ElasticsearchPersistentEntity#getScoreProperty()} to determine whether the
-	 * current property is the score property of that {@link ElasticsearchPersistentEntity} under consideration.
+	 * rather call {@link ElasticsearchPersistentEntity#getScoreProperty()} to determine whether the current property is
+	 * the score property of that {@link ElasticsearchPersistentEntity} under consideration.
 	 *
 	 * @return
 	 * @since 3.1
@@ -64,6 +65,14 @@ public interface ElasticsearchPersistentProperty extends PersistentProperty<Elas
 	 */
 	boolean isParentProperty();
 
+	/**
+	 * Returns whether the current property is a {@link SeqNoPrimaryTerm} property.
+	 *
+	 * @return true if the type is {@link SeqNoPrimaryTerm}
+	 * @since 4.0
+	 */
+	boolean isSeqNoPrimaryTermProperty();
+
 	/**
 	 * @return true if an {@link ElasticsearchPersistentPropertyConverter} is available for this instance.
 	 * @since 4.0
@@ -77,6 +86,14 @@ public interface ElasticsearchPersistentProperty extends PersistentProperty<Elas
 	@Nullable
 	ElasticsearchPersistentPropertyConverter getPropertyConverter();
 
+	/**
+	 * Returns true if the property may be read.
+	 *
+	 * @return true if readable, false otherwise
+	 * @since 4.0
+	 */
+	boolean isReadable();
+
 	enum PropertyToFieldNameConverter implements Converter<ElasticsearchPersistentProperty, String> {
 
 		INSTANCE;

src/main/java/org/springframework/data/elasticsearch/core/mapping/SimpleElasticsearchMappingContext.java

@@ -37,6 +37,11 @@ public class SimpleElasticsearchMappingContext
 
 	private @Nullable ApplicationContext context;
 
+	@Override
+	protected boolean shouldCreatePersistentEntityFor(TypeInformation<?> type) {
+		return !ElasticsearchSimpleTypes.HOLDER.isSimpleType(type.getType());
+	}
+
 	@Override
 	protected <T> SimpleElasticsearchPersistentEntity<?> createPersistentEntity(TypeInformation<T> typeInformation) {
 		SimpleElasticsearchPersistentEntity<T> persistentEntity = new SimpleElasticsearchPersistentEntity<>(

src/main/java/org/springframework/data/elasticsearch/core/mapping/SimpleElasticsearchPersistentEntity.java

@@ -23,6 +23,8 @@ import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.atomic.AtomicReference;
 
 import org.elasticsearch.index.VersionType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.springframework.beans.BeansException;
 import org.springframework.context.ApplicationContext;
 import org.springframework.context.ApplicationContextAware;
@@ -53,10 +55,13 @@ import org.springframework.util.Assert;
  * @author Sascha Woo
  * @author Ivan Greene
  * @author Peter-Josef Meisch
+ * @author Roman Puchkovskiy
  */
 public class SimpleElasticsearchPersistentEntity<T> extends BasicPersistentEntity<T, ElasticsearchPersistentProperty>
 		implements ElasticsearchPersistentEntity<T>, ApplicationContextAware {
 
+	private static final Logger LOGGER = LoggerFactory.getLogger(SimpleElasticsearchPersistentEntity.class);
+
 	private final StandardEvaluationContext context;
 	private final SpelExpressionParser parser;
 
@@ -70,6 +75,7 @@ public class SimpleElasticsearchPersistentEntity<T> extends BasicPersistentEntit
 	private @Nullable String parentType;
 	private @Nullable ElasticsearchPersistentProperty parentIdProperty;
 	private @Nullable ElasticsearchPersistentProperty scoreProperty;
+	private @Nullable ElasticsearchPersistentProperty seqNoPrimaryTermProperty;
 	private @Nullable String settingPath;
 	private @Nullable VersionType versionType;
 	private boolean createIndexAndMapping;
@@ -231,6 +237,36 @@ public class SimpleElasticsearchPersistentEntity<T> extends BasicPersistentEntit
 
 			this.scoreProperty = property;
 		}
+
+		if (property.isSeqNoPrimaryTermProperty()) {
+
+			ElasticsearchPersistentProperty seqNoPrimaryTermProperty = this.seqNoPrimaryTermProperty;
+
+			if (seqNoPrimaryTermProperty != null) {
+				throw new MappingException(String.format(
+						"Attempt to add SeqNoPrimaryTerm property %s but already have property %s registered "
+								+ "as SeqNoPrimaryTerm property. Check your entity configuration!",
+						property.getField(), seqNoPrimaryTermProperty.getField()));
+			}
+
+			this.seqNoPrimaryTermProperty = property;
+
+			if (hasVersionProperty()) {
+				warnAboutBothSeqNoPrimaryTermAndVersionProperties();
+			}
+		}
+
+		if (property.isVersionProperty()) {
+			if (hasSeqNoPrimaryTermProperty()) {
+				warnAboutBothSeqNoPrimaryTermAndVersionProperties();
+			}
+		}
+	}
+
+	private void warnAboutBothSeqNoPrimaryTermAndVersionProperties() {
+		LOGGER.warn(
+				"Both SeqNoPrimaryTerm and @Version properties are defined on {}. Version will not be sent in index requests when seq_no is sent!",
+				getType());
 	}
 
 	/*
@@ -262,4 +298,15 @@ public class SimpleElasticsearchPersistentEntity<T> extends BasicPersistentEntit
 			return propertyRef.get();
 		});
 	}
+
+	@Override
+	public boolean hasSeqNoPrimaryTermProperty() {
+		return seqNoPrimaryTermProperty != null;
+	}
+
+	@Override
+	@Nullable
+	public ElasticsearchPersistentProperty getSeqNoPrimaryTermProperty() {
+		return seqNoPrimaryTermProperty;
+	}
 }

src/main/java/org/springframework/data/elasticsearch/core/mapping/SimpleElasticsearchPersistentProperty.java

@@ -17,14 +17,17 @@ package org.springframework.data.elasticsearch.core.mapping;
 
 import java.time.temporal.TemporalAccessor;
 import java.util.Arrays;
+import java.util.Date;
 import java.util.List;
 
 import org.springframework.data.elasticsearch.annotations.DateFormat;
 import org.springframework.data.elasticsearch.annotations.Field;
 import org.springframework.data.elasticsearch.annotations.FieldType;
+import org.springframework.data.elasticsearch.annotations.MultiField;
 import org.springframework.data.elasticsearch.annotations.Parent;
 import org.springframework.data.elasticsearch.annotations.Score;
 import org.springframework.data.elasticsearch.core.convert.ElasticsearchDateConverter;
+import org.springframework.data.elasticsearch.core.query.SeqNoPrimaryTerm;
 import org.springframework.data.mapping.Association;
 import org.springframework.data.mapping.MappingException;
 import org.springframework.data.mapping.PersistentEntity;
@@ -43,6 +46,7 @@ import org.springframework.util.StringUtils;
  * @author Sascha Woo
  * @author Oliver Gierke
  * @author Peter-Josef Meisch
+ * @author Roman Puchkovskiy
  */
 public class SimpleElasticsearchPersistentProperty extends
 		AnnotationBasedPersistentProperty<ElasticsearchPersistentProperty> implements ElasticsearchPersistentProperty {
@@ -52,6 +56,7 @@ public class SimpleElasticsearchPersistentProperty extends
 	private final boolean isScore;
 	private final boolean isParent;
 	private final boolean isId;
+	private final boolean isSeqNoPrimaryTerm;
 	private final @Nullable String annotatedFieldName;
 	@Nullable private ElasticsearchPersistentPropertyConverter propertyConverter;
 
@@ -64,6 +69,7 @@ public class SimpleElasticsearchPersistentProperty extends
 		this.isId = super.isIdProperty() || SUPPORTED_ID_PROPERTY_NAMES.contains(getFieldName());
 		this.isScore = isAnnotationPresent(Score.class);
 		this.isParent = isAnnotationPresent(Parent.class);
+		this.isSeqNoPrimaryTerm = SeqNoPrimaryTerm.class.isAssignableFrom(getRawType());
 
 		if (isVersionProperty() && !getType().equals(Long.class)) {
 			throw new MappingException(String.format("Version property %s must be of type Long!", property.getName()));
@@ -78,6 +84,10 @@ public class SimpleElasticsearchPersistentProperty extends
 			throw new MappingException(String.format("Parent property %s must be of type String!", property.getName()));
 		}
 
+		if (isAnnotationPresent(Field.class) && isAnnotationPresent(MultiField.class)) {
+			throw new MappingException("@Field annotation must not be used on a @MultiField property.");
+		}
+
 		initDateConverter();
 	}
 
@@ -92,56 +102,93 @@ public class SimpleElasticsearchPersistentProperty extends
 		return propertyConverter;
 	}
 
+	@Override
+	public boolean isWritable() {
+		return super.isWritable() && !isSeqNoPrimaryTermProperty();
+	}
+
+	@Override
+	public boolean isReadable() {
+		return !isTransient() && !isSeqNoPrimaryTermProperty();
+	}
+
 	/**
 	 * Initializes an {@link ElasticsearchPersistentPropertyConverter} if this property is annotated as a Field with type
 	 * {@link FieldType#Date}, has a {@link DateFormat} set and if the type of the property is one of the Java8 temporal
-	 * classes.
+	 * classes or java.util.Date.
 	 */
 	private void initDateConverter() {
 		Field field = findAnnotation(Field.class);
-		if (field != null && field.type() == FieldType.Date && TemporalAccessor.class.isAssignableFrom(getType())) {
+		Class<?> actualType = getActualType();
+		boolean isTemporalAccessor = TemporalAccessor.class.isAssignableFrom(actualType);
+		boolean isDate = Date.class.isAssignableFrom(actualType);
+
+		if (field != null && (field.type() == FieldType.Date || field.type() == FieldType.Date_Nanos)
+				&& (isTemporalAccessor || isDate)) {
 			DateFormat dateFormat = field.format();
 
-			ElasticsearchDateConverter converter = null;
+			if (dateFormat == DateFormat.none) {
+				throw new MappingException(
+						String.format("Property %s is annotated with FieldType.%s but has no DateFormat defined",
+								getOwner().getType().getSimpleName() + "." + getName(), field.type().name()));
+			}
+
+			ElasticsearchDateConverter converter;
 
 			if (dateFormat == DateFormat.custom) {
 				String pattern = field.pattern();
 
-				if (StringUtils.hasLength(pattern)) {
-					converter = ElasticsearchDateConverter.of(pattern);
+				if (!StringUtils.hasLength(pattern)) {
+					throw new MappingException(
+							String.format("Property %s is annotated with FieldType.%s and a custom format but has no pattern defined",
+									getOwner().getType().getSimpleName() + "." + getName(), field.type().name()));
 				}
-			} else if (dateFormat != DateFormat.none) {
+
+				converter = ElasticsearchDateConverter.of(pattern);
+			} else {
 				converter = ElasticsearchDateConverter.of(dateFormat);
 			}
 
-			if (converter != null) {
-				ElasticsearchDateConverter dateConverter = converter;
-				propertyConverter = new ElasticsearchPersistentPropertyConverter() {
-					@Override
-					public String write(Object property) {
-						return dateConverter.format((TemporalAccessor) property);
-					}
+			propertyConverter = new ElasticsearchPersistentPropertyConverter() {
+				final ElasticsearchDateConverter dateConverter = converter;
 
-					@SuppressWarnings("unchecked")
-					@Override
-					public Object read(String s) {
-						return dateConverter.parse(s, (Class<? extends TemporalAccessor>) getType());
+				@Override
+				public String write(Object property) {
+					if (isTemporalAccessor && TemporalAccessor.class.isAssignableFrom(property.getClass())) {
+						return dateConverter.format((TemporalAccessor) property);
+					} else if (isDate && Date.class.isAssignableFrom(property.getClass())) {
+						return dateConverter.format((Date) property);
+					} else {
+						return property.toString();
 					}
-				};
-			}
+				}
+
+				@SuppressWarnings("unchecked")
+				@Override
+				public Object read(String s) {
+					if (isTemporalAccessor) {
+						return dateConverter.parse(s, (Class<? extends TemporalAccessor>) actualType);
+					} else { // must be date
+						return dateConverter.parse(s);
+					}
+				}
+			};
 		}
 	}
 
+	@SuppressWarnings("ConstantConditions")
 	@Nullable
 	private String getAnnotatedFieldName() {
 
-		if (isAnnotationPresent(Field.class)) {
+		String name = null;
 
-			String name = findAnnotation(Field.class).name();
-			return StringUtils.hasText(name) ? name : null;
+		if (isAnnotationPresent(Field.class)) {
+			name = findAnnotation(Field.class).name();
+		} else if (isAnnotationPresent(MultiField.class)) {
+			name = findAnnotation(MultiField.class).mainField().name();
 		}
 
-		return null;
+		return StringUtils.hasText(name) ? name : null;
 	}
 
 	/*
@@ -197,4 +244,13 @@ public class SimpleElasticsearchPersistentProperty extends
 	public boolean isParentProperty() {
 		return isParent;
 	}
+
+	/*
+	 * (non-Javadoc)
+	 * @see org.springframework.data.elasticsearch.core.mapping.ElasticsearchPersistentProperty#isSeqNoPrimaryTermProperty()
+	 */
+	@Override
+	public boolean isSeqNoPrimaryTermProperty() {
+		return isSeqNoPrimaryTerm;
+	}
 }

src/main/java/org/springframework/data/elasticsearch/core/query/Criteria.java

@@ -393,11 +393,6 @@ public class Criteria {
 	}
 
 	private List<Object> toCollection(Object... values) {
-		if (values.length == 0 || (values.length > 1 && values[1] instanceof Collection)) {
-			throw new InvalidDataAccessApiUsageException(
-					"At least one element " + (values.length > 0 ? ("of argument of type " + values[1].getClass().getName()) : "")
-							+ " has to be present.");
-		}
 		return Arrays.asList(values);
 	}
 

src/main/java/org/springframework/data/elasticsearch/core/query/Field.java

@@ -15,6 +15,9 @@
  */
 package org.springframework.data.elasticsearch.core.query;
 
+import org.springframework.data.elasticsearch.annotations.FieldType;
+import org.springframework.lang.Nullable;
+
 /**
  * Defines a Field that can be used within a Criteria.
  *
@@ -27,4 +30,15 @@ public interface Field {
 	void setName(String name);
 
 	String getName();
+
+	/**
+	 * @param fieldType sets the field's type
+	 */
+	void setFieldType(FieldType fieldType);
+
+	/**
+	 * @return The annotated FieldType of the field
+	 */
+	@Nullable
+	FieldType getFieldType();
 }

src/main/java/org/springframework/data/elasticsearch/core/query/IndexQuery.java

@@ -23,8 +23,8 @@ import org.springframework.lang.Nullable;
  * @author Rizwan Idrees
  * @author Mohsin Husen
  * @author Peter-Josef Meisch
+ * @author Roman Puchkovskiy
  */
-
 public class IndexQuery {
 
 	@Nullable private String id;
@@ -32,6 +32,8 @@ public class IndexQuery {
 	@Nullable private Long version;
 	@Nullable private String source;
 	@Nullable private String parentId;
+	@Nullable private Long seqNo;
+	@Nullable private Long primaryTerm;
 
 	@Nullable
 	public String getId() {
@@ -87,4 +89,22 @@ public class IndexQuery {
 	public void setParentId(String parentId) {
 		this.parentId = parentId;
 	}
+
+	@Nullable
+	public Long getSeqNo() {
+		return seqNo;
+	}
+
+	public void setSeqNo(Long seqNo) {
+		this.seqNo = seqNo;
+	}
+
+	@Nullable
+	public Long getPrimaryTerm() {
+		return primaryTerm;
+	}
+
+	public void setPrimaryTerm(Long primaryTerm) {
+		this.primaryTerm = primaryTerm;
+	}
 }