iSharkFly-Docs/java-tutorials
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[JAVA-33624] Split-or-move-apache-kafka-2-module (#16429)

Browse Source
This commit is contained in:
vunamtien
2024-04-20 20:09:50 +07:00
committed by GitHub
parent 57f2c2ef23
commit 7163e8b266
5 changed files with 20 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apache-kafka-3/README.md
+2
View File
@@ -7,4 +7,6 @@ You can build the project from the command line using: *mvn clean install*, or i
### Relevant Articles:
- [Commit Offsets in Kafka](https://www.baeldung.com/kafka-commit-offsets)
- [Get Last N Messages in Apache Kafka Topic](https://www.baeldung.com/java-apache-kafka-get-last-n-messages)
- [Get Partition Count for a Topic in Kafka](https://www.baeldung.com/java-kafka-partition-count-topic)
apache-kafka-3/pom.xml
+14
View File
@@ -34,11 +34,25 @@
<artifactId>jackson-databind</artifactId>
<version>${jackson.databind.version}</version>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>kafka</artifactId>
<version>${testcontainers-kafka.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>junit-jupiter</artifactId>
<version>${testcontainers-jupiter.version}</version>
<scope>test</scope>
</dependency>
</dependencies>
<properties>
<kafka.version>3.6.1</kafka.version>
<jackson.databind.version>2.15.2</jackson.databind.version>
<testcontainers-kafka.version>1.19.3</testcontainers-kafka.version>
<testcontainers-jupiter.version>1.19.3</testcontainers-jupiter.version>
</properties>
<profiles>
<profile>
apache-kafka-3/src/test/java/com/baeldung/kafka/KafaConsumeLastNMessagesLiveTest.java
+138
View File
@@ -0,0 +1,138 @@
package com.baeldung.kafka;
import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.ProducerConfig;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.apache.kafka.common.TopicPartition;
import org.apache.kafka.common.serialization.StringDeserializer;
import org.apache.kafka.common.serialization.StringSerializer;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.testcontainers.containers.KafkaContainer;
import org.testcontainers.junit.jupiter.Container;
import org.testcontainers.junit.jupiter.Testcontainers;
import org.testcontainers.utility.DockerImageName;
import java.time.Duration;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
import java.util.concurrent.ExecutionException;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
@Testcontainers
public class KafaConsumeLastNMessagesLiveTest {
private static String TOPIC1 = "baeldung-github";
private static String TOPIC2 = "baeldung-blog";
private static String MESSAGE_KEY = "message";
private static KafkaProducer<String, String> producer;
private static KafkaConsumer<String, String> consumer;
private static KafkaProducer<String, String> transactionalProducer;
@Container
private static final KafkaContainer KAFKA_CONTAINER = new KafkaContainer(DockerImageName.parse("confluentinc/cp-kafka:latest"));
@BeforeAll
static void setup() {
KAFKA_CONTAINER.addExposedPort(9092);
Properties producerProperties = new Properties();
producerProperties.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, KAFKA_CONTAINER.getBootstrapServers());
producerProperties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
producerProperties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
Properties consumerProperties = new Properties();
consumerProperties.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, KAFKA_CONTAINER.getBootstrapServers());
consumerProperties.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());
consumerProperties.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());
consumerProperties.put(ConsumerConfig.GROUP_ID_CONFIG, "ConsumerGroup1");
Properties transactionalProducerProprieties = new Properties();
transactionalProducerProprieties.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, KAFKA_CONTAINER.getBootstrapServers());
transactionalProducerProprieties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
transactionalProducerProprieties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
transactionalProducerProprieties.put(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG, "true");
transactionalProducerProprieties.put(ProducerConfig.TRANSACTIONAL_ID_CONFIG, "prod-0");
producer = new KafkaProducer<>(producerProperties);
consumer = new KafkaConsumer<>(consumerProperties);
transactionalProducer = new KafkaProducer<>(transactionalProducerProprieties);
}
@AfterAll
static void destroy() {
KAFKA_CONTAINER.stop();
}
@Test
void whenSeekingKafkaTopicCursorToEnd_consumerRetrievesOnlyDesiredNumberOfMessages() throws ExecutionException, InterruptedException {
int messagesInTopic = 100;
int messagesToRetrieve = 20;
for (int i = 0; i < messagesInTopic; i++) {
producer.send(new ProducerRecord<>(TOPIC1, null, MESSAGE_KEY, String.valueOf(i)))
.get();
}
TopicPartition partition = new TopicPartition(TOPIC1, 0);
List<TopicPartition> partitions = new ArrayList<>();
partitions.add(partition);
consumer.assign(partitions);
consumer.seekToEnd(partitions);
long startIndex = consumer.position(partition) - messagesToRetrieve;
consumer.seek(partition, startIndex);
ConsumerRecords<String, String> records = consumer.poll(Duration.ofMinutes(1));
int recordsReceived = 0;
for (ConsumerRecord<String, String> record : records) {
assertEquals(MESSAGE_KEY, record.key());
assertTrue(Integer.parseInt(record.value()) >= (messagesInTopic - messagesToRetrieve));
recordsReceived++;
}
assertEquals(messagesToRetrieve, recordsReceived);
}
@Test
void havingTransactionalProducer_whenSeekingKafkaTopicCursorToEnd_consumerRetrievesLessMessages() throws ExecutionException, InterruptedException {
int messagesInTopic = 100;
int messagesToRetrieve = 20;
transactionalProducer.initTransactions();
for (int i = 0; i < messagesInTopic; i++) {
transactionalProducer.beginTransaction();
transactionalProducer.send(new ProducerRecord<>(TOPIC2, null, MESSAGE_KEY, String.valueOf(i)))
.get();
transactionalProducer.commitTransaction();
}
TopicPartition partition = new TopicPartition(TOPIC2, 0);
List<TopicPartition> partitions = new ArrayList<>();
partitions.add(partition);
consumer.assign(partitions);
consumer.seekToEnd(partitions);
long startIndex = consumer.position(partition) - messagesToRetrieve;
consumer.seek(partition, startIndex);
ConsumerRecords<String, String> records = consumer.poll(Duration.ofMinutes(1));
int recordsReceived = 0;
for (ConsumerRecord<String, String> record : records) {
assertEquals(MESSAGE_KEY, record.key());
assertTrue(Integer.parseInt(record.value()) >= (messagesInTopic - messagesToRetrieve));
recordsReceived++;
}
assertTrue(messagesToRetrieve > recordsReceived);
}
}
apache-kafka-3/src/test/java/com/baeldung/kafka/KafkaCountPartitionsLiveTest.java
+86
View File
@@ -0,0 +1,86 @@
package com.baeldung.kafka;
import org.apache.kafka.clients.admin.AdminClient;
import org.apache.kafka.clients.admin.DescribeTopicsResult;
import org.apache.kafka.clients.admin.TopicDescription;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.ProducerConfig;
import org.apache.kafka.common.KafkaFuture;
import org.apache.kafka.common.PartitionInfo;
import org.apache.kafka.common.serialization.StringSerializer;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.testcontainers.containers.KafkaContainer;
import org.testcontainers.junit.jupiter.Container;
import org.testcontainers.junit.jupiter.Testcontainers;
import org.testcontainers.utility.DockerImageName;
import java.io.IOException;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.ExecutionException;
@Testcontainers
public class KafkaCountPartitionsLiveTest {
@Container
private static final KafkaContainer KAFKA_CONTAINER = new KafkaContainer(DockerImageName.parse("confluentinc/cp-kafka:latest"));
private final static String TOPIC = "baeldung-kafka-github";
private final static Integer PARTITION_NUMBER = 3;
private static KafkaProducer<String, String> producer;
@BeforeAll
static void setup() throws IOException, InterruptedException {
KAFKA_CONTAINER.addExposedPort(9092);
KAFKA_CONTAINER.execInContainer(
"/bin/sh",
"-c",
"/usr/bin/kafka-topics " +
"--bootstrap-server localhost:9092 " +
"--create " +
"--replication-factor 1 " +
"--partitions " + PARTITION_NUMBER + " " +
"--topic " + TOPIC
);
Properties producerProperties = new Properties();
producerProperties.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, KAFKA_CONTAINER.getBootstrapServers());
producerProperties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
producerProperties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
producer = new KafkaProducer<>(producerProperties);
}
@AfterAll
static void destroy() {
KAFKA_CONTAINER.stop();
}
@Test
void testPartitionsForTopic_isEqualToActualNumberAssignedDuringCreation() {
List<PartitionInfo> info = producer.partitionsFor(TOPIC);
Assertions.assertEquals(PARTITION_NUMBER, info.size());
}
@Test
void testTopicPartitionDescription_isEqualToActualNumberAssignedDuringCreation() {
Properties props = new Properties();
props.put("bootstrap.servers", KAFKA_CONTAINER.getBootstrapServers());
props.put("client.id","java-admin-client");
props.put("request.timeout.ms", 3000);
props.put("connections.max.idle.ms", 5000);
try(AdminClient client = AdminClient.create(props)){
DescribeTopicsResult describeTopicsResult = client.describeTopics(Collections.singletonList(TOPIC));
Map<String, KafkaFuture<TopicDescription>> values = describeTopicsResult.values();
KafkaFuture<TopicDescription> topicDescription = values.get(TOPIC);
Assertions.assertEquals(PARTITION_NUMBER, topicDescription.get().partitions().size());
} catch (ExecutionException | InterruptedException e) {
throw new RuntimeException(e);
}
}
}