Release version 5.3.2 (2024.0.2).

See #2932

Jenkinsfile

@@ -39,9 +39,11 @@ pipeline {
 
 			steps {
 				script {
-					docker.image(p['docker.java.main.image']).inside(p['docker.java.inside.docker']) {
-						sh "PROFILE=none JENKINS_USER_NAME=${p['jenkins.user.name']} ci/verify.sh"
-						sh "JENKINS_USER_NAME=${p['jenkins.user.name']} ci/clean.sh"
+					docker.withRegistry(p['docker.proxy.registry'], p['docker.proxy.credentials']) {
+						docker.image(p['docker.java.main.image']).inside(p['docker.java.inside.docker']) {
+							sh "PROFILE=none JENKINS_USER_NAME=${p['jenkins.user.name']} ci/verify.sh"
+							sh "JENKINS_USER_NAME=${p['jenkins.user.name']} ci/clean.sh"
+						}
 					}
 				}
 			}
@@ -68,9 +70,11 @@ pipeline {
 					}
 					steps {
 						script {
-							docker.image(p['docker.java.next.image']).inside(p['docker.java.inside.docker']) {
-								sh "PROFILE=none JENKINS_USER_NAME=${p['jenkins.user.name']} ci/verify.sh"
-								sh "JENKINS_USER_NAME=${p['jenkins.user.name']} ci/clean.sh"
+							docker.withRegistry(p['docker.proxy.registry'], p['docker.proxy.credentials']) {
+								docker.image(p['docker.java.next.image']).inside(p['docker.java.inside.docker']) {
+									sh "PROFILE=none JENKINS_USER_NAME=${p['jenkins.user.name']} ci/verify.sh"
+									sh "JENKINS_USER_NAME=${p['jenkins.user.name']} ci/clean.sh"
+								}
 							}
 						}
 					}
@@ -97,19 +101,21 @@ pipeline {
 			}
 			steps {
 				script {
-					docker.image(p['docker.java.main.image']).inside(p['docker.java.inside.basic']) {
-							sh 'MAVEN_OPTS="-Duser.name=' + "${p['jenkins.user.name']}" + ' -Duser.home=/tmp/jenkins-home" ' +
-								"DEVELOCITY_CACHE_USERNAME=${DEVELOCITY_CACHE_USR} " +
-								"DEVELOCITY_CACHE_PASSWORD=${DEVELOCITY_CACHE_PSW} " +
-								"GRADLE_ENTERPRISE_ACCESS_KEY=${DEVELOCITY_ACCESS_KEY} " +
-								"./mvnw -s settings.xml -Pci,artifactory -Dmaven.repo.local=/tmp/jenkins-home/.m2/spring-data-elasticsearch-non-root " +
-								"-Dartifactory.server=${p['artifactory.url']} " +
-								"-Dartifactory.username=${ARTIFACTORY_USR} " +
-								"-Dartifactory.password=${ARTIFACTORY_PSW} " +
-								"-Dartifactory.staging-repository=${p['artifactory.repository.snapshot']} " +
-								"-Dartifactory.build-name=spring-data-elasticsearch " +
-								"-Dartifactory.build-number=spring-data-elasticsearch-${BRANCH_NAME}-build-${BUILD_NUMBER} " +
-								"-Dmaven.test.skip=true clean deploy -U -B"
+					docker.withRegistry(p['docker.proxy.registry'], p['docker.proxy.credentials']) {
+						docker.image(p['docker.java.main.image']).inside(p['docker.java.inside.basic']) {
+								sh 'MAVEN_OPTS="-Duser.name=' + "${p['jenkins.user.name']}" + ' -Duser.home=/tmp/jenkins-home" ' +
+									"DEVELOCITY_CACHE_USERNAME=${DEVELOCITY_CACHE_USR} " +
+									"DEVELOCITY_CACHE_PASSWORD=${DEVELOCITY_CACHE_PSW} " +
+									"GRADLE_ENTERPRISE_ACCESS_KEY=${DEVELOCITY_ACCESS_KEY} " +
+									"./mvnw -s settings.xml -Pci,artifactory -Dmaven.repo.local=/tmp/jenkins-home/.m2/spring-data-elasticsearch-non-root " +
+									"-Dartifactory.server=${p['artifactory.url']} " +
+									"-Dartifactory.username=${ARTIFACTORY_USR} " +
+									"-Dartifactory.password=${ARTIFACTORY_PSW} " +
+									"-Dartifactory.staging-repository=${p['artifactory.repository.snapshot']} " +
+									"-Dartifactory.build-name=spring-data-elasticsearch " +
+									"-Dartifactory.build-number=spring-data-elasticsearch-${BRANCH_NAME}-build-${BUILD_NUMBER} " +
+									"-Dmaven.test.skip=true clean deploy -U -B"
+						}
 					}
 				}
 			}

ci/pipeline.properties

@@ -3,8 +3,8 @@ java.main.tag=17.0.9_9-jdk-focal
 java.next.tag=21.0.1_12-jdk-jammy
 
 # Docker container images - standard
-docker.java.main.image=harbor-repo.vmware.com/dockerhub-proxy-cache/library/eclipse-temurin:${java.main.tag}
-docker.java.next.image=harbor-repo.vmware.com/dockerhub-proxy-cache/library/eclipse-temurin:${java.next.tag}
+docker.java.main.image=library/eclipse-temurin:${java.main.tag}
+docker.java.next.image=library/eclipse-temurin:${java.next.tag}
 
 # Supported versions of MongoDB
 docker.mongodb.4.4.version=4.4.25
@@ -14,6 +14,7 @@ docker.mongodb.7.0.version=7.0.2
 
 # Supported versions of Redis
 docker.redis.6.version=6.2.13
+docker.redis.7.version=7.2.4
 
 # Supported versions of Cassandra
 docker.cassandra.3.version=3.11.16
@@ -25,6 +26,8 @@ docker.java.inside.docker=-u root -v /var/run/docker.sock:/var/run/docker.sock -
 # Credentials
 docker.registry=
 docker.credentials=hub.docker.com-springbuildmaster
+docker.proxy.registry=https://docker-hub.usw1.packages.broadcom.com
+docker.proxy.credentials=usw1_packages_broadcom_com-jenkins-token
 artifactory.credentials=02bd1690-b54f-4c9f-819d-a77cb7a9822c
 artifactory.url=https://repo.spring.io
 artifactory.repository.snapshot=libs-snapshot-local

pom.xml

@@ -5,12 +5,12 @@
 
 	<groupId>org.springframework.data</groupId>
 	<artifactId>spring-data-elasticsearch</artifactId>
-	<version>5.3.1</version>
+	<version>5.3.2</version>
 
 	<parent>
 		<groupId>org.springframework.data.build</groupId>
 		<artifactId>spring-data-parent</artifactId>
-		<version>3.3.1</version>
+		<version>3.3.2</version>
 	</parent>
 
 	<name>Spring Data Elasticsearch</name>
@@ -18,7 +18,7 @@
 	<url>https://github.com/spring-projects/spring-data-elasticsearch</url>
 
 	<properties>
-		<springdata.commons>3.3.1</springdata.commons>
+		<springdata.commons>3.3.2</springdata.commons>
 
 		<!-- version of the ElasticsearchClient	-->
 		<elasticsearch-java>8.13.4</elasticsearch-java>

src/main/antora/modules/ROOT/pages/migration-guides/migration-guide-5.2-5.3.adoc

@@ -5,7 +5,10 @@ This section describes breaking changes from version 5.2.x to 5.3.x and how remo
 
 [[elasticsearch-migration-guide-5.2-5.3.breaking-changes]]
 == Breaking Changes
-
+During the parameter replacement in `@Query` annotated repository methods previous versions wrote the String _"null"_ into the query that was sent to Elasticsearch 
+when the actual parameter value was `null`. As Elasticsearch does not store `null` values, this behaviour could lead to problems, for example whent the fields to be 
+searched contains the string `"null"`. In Version 5.3 a `null` value in a parameter will cause a `ConversionException` to be thrown. If you are using `"null"` as the 
+`null_value` defined in a field mapping, then pass that string into the query instead of a Java `null`.
 
 [[elasticsearch-migration-guide-5.2-5.3.deprecations]]
 == Deprecations

src/main/java/org/springframework/data/elasticsearch/client/elc/ReactiveElasticsearchTemplate.java

@@ -395,7 +395,28 @@ public class ReactiveElasticsearchTemplate extends AbstractReactiveElasticsearch
 			Function<PitSearchAfter, Publisher<? extends ResponseBody<EntityAsMap>>> resourceClosure = psa -> {
 
 				baseQuery.setPointInTime(new Query.PointInTime(psa.getPit(), pitKeepAlive));
-				baseQuery.addSort(Sort.by("_shard_doc"));
+
+				// only add _shard_doc if there is not a field_collapse and a sort with the same name
+				boolean addShardDoc = true;
+
+				if (query instanceof NativeQuery nativeQuery && nativeQuery.getFieldCollapse() != null) {
+					var field = nativeQuery.getFieldCollapse().field();
+
+					if (nativeQuery.getSortOptions().stream()
+							.anyMatch(sortOptions -> sortOptions.isField() && sortOptions.field().field().equals(field))) {
+						addShardDoc = false;
+					}
+
+					if (query.getSort() != null
+							&& query.getSort().stream().anyMatch(order -> order.getProperty().equals(field))) {
+						addShardDoc = false;
+					}
+				}
+
+				if (addShardDoc) {
+					baseQuery.addSort(Sort.by("_shard_doc"));
+				}
+
 				SearchRequest firstSearchRequest = requestConverter.searchRequest(baseQuery, routingResolver.getRouting(),
 						clazz, index, false, true);
 

src/main/java/org/springframework/data/elasticsearch/client/elc/RequestConverter.java

@@ -1477,8 +1477,8 @@ class RequestConverter extends AbstractQueryProcessor {
 		if (query instanceof NativeQuery nativeQuery) {
 			prepareNativeSearch(nativeQuery, builder);
 		}
-		// query.getSort() must be checked after prepareNativeSearch as this already might hav a sort set that must have
-		// higher priority
+		// query.getSort() must be checked after prepareNativeSearch as this already might have a sort set
+		// that must have higher priority
 		if (query.getSort() != null) {
 			List<SortOptions> sortOptions = getSortOptions(query.getSort(), persistentEntity);
 
@@ -1500,7 +1500,15 @@ class RequestConverter extends AbstractQueryProcessor {
 		}
 
 		if (!isEmpty(query.getSearchAfter())) {
-			builder.searchAfter(query.getSearchAfter().stream().map(TypeUtils::toFieldValue).toList());
+			var fieldValues = query.getSearchAfter().stream().map(TypeUtils::toFieldValue).toList();
+
+			// when there is a field collapse on a native query, and we have a search_after, then the search_after
+			// must only have one entry
+			if (query instanceof NativeQuery nativeQuery && nativeQuery.getFieldCollapse() != null) {
+				builder.searchAfter(fieldValues.get(0));
+			} else {
+				builder.searchAfter(fieldValues);
+			}
 		}
 
 		query.getRescorerQueries().forEach(rescorerQuery -> builder.rescore(getRescore(rescorerQuery)));

src/main/resources/notice.txt

@@ -1,4 +1,4 @@
-Spring Data Elasticsearch 5.3.1 (2024.0.1)
+Spring Data Elasticsearch 5.3.2 (2024.0.2)
 Copyright (c) [2013-2022] Pivotal Software, Inc.
 
 This product is licensed to you under the Apache License, Version 2.0 (the "License").
@@ -23,3 +23,4 @@ conditions of the subcomponent's license, as noted in the LICENSE file.
 
 
 
+

src/test/java/org/springframework/data/elasticsearch/repository/support/SimpleReactiveElasticsearchRepositoryELCIntegrationTests.java

@@ -15,14 +15,22 @@
  */
 package org.springframework.data.elasticsearch.repository.support;
 
+import co.elastic.clients.elasticsearch.core.search.FieldCollapse;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.Import;
+import org.springframework.data.domain.Pageable;
+import org.springframework.data.domain.Sort;
+import org.springframework.data.elasticsearch.client.elc.NativeQuery;
+import org.springframework.data.elasticsearch.client.elc.Queries;
 import org.springframework.data.elasticsearch.junit.jupiter.ReactiveElasticsearchTemplateConfiguration;
 import org.springframework.data.elasticsearch.repositories.custommethod.QueryParameter;
 import org.springframework.data.elasticsearch.repository.config.EnableReactiveElasticsearchRepositories;
 import org.springframework.data.elasticsearch.utils.IndexNameProvider;
 import org.springframework.test.context.ContextConfiguration;
+import reactor.test.StepVerifier;
 
 /**
  * @author Peter-Josef Meisch
@@ -51,4 +59,33 @@ public class SimpleReactiveElasticsearchRepositoryELCIntegrationTests
 		}
 	}
 
+	/**
+	 * search_after is used by the reactive search operation, it normally always adds _shard_doc as a tiebreaker sort
+	 * parameter. This must not be done when a collapse field is used as sort field, as in that case the collapse field
+	 * must be the only sort field.
+	 */
+	@Test // #2935
+	@DisplayName("should use collapse_field for search_after in pit search")
+	void shouldUseCollapseFieldForSearchAfterI() {
+		var entity = new SampleEntity();
+		entity.setId("42");
+		entity.setMessage("m");
+		entity.setKeyword("kw");
+		repository.save(entity).block();
+
+		var query = NativeQuery.builder()
+				.withQuery(Queries.matchAllQueryAsQuery())
+				.withPageable(Pageable.unpaged())
+				.withFieldCollapse(FieldCollapse.of(fcb -> fcb
+						.field("keyword")))
+				.withSort(Sort.by("keyword"))
+				.build();
+
+		operations.search(query, SampleEntity.class)
+				.as(StepVerifier::create)
+				.expectNextCount(1)
+				.verifyComplete();
+	}
+
+
 }