JAVA-6390: Move kafka articles from libraries-data-3 to new module
apache-kafka
This commit is contained in:
sampadawagde
@@ -0,0 +1,104 @@
|
||||
package com.baeldung.flink;
|
||||
|
||||
import com.baeldung.flink.model.Backup;
|
||||
import com.baeldung.flink.model.InputMessage;
|
||||
import com.baeldung.flink.operator.BackupAggregator;
|
||||
import com.baeldung.flink.operator.InputMessageTimestampAssigner;
|
||||
import com.baeldung.flink.schema.BackupSerializationSchema;
|
||||
import com.baeldung.flink.schema.InputMessageDeserializationSchema;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
|
||||
import org.apache.commons.collections.ListUtils;
|
||||
import org.apache.flink.api.common.serialization.DeserializationSchema;
|
||||
import org.apache.flink.api.common.serialization.SerializationSchema;
|
||||
import org.apache.flink.streaming.api.TimeCharacteristic;
|
||||
import org.apache.flink.streaming.api.datastream.DataStreamSource;
|
||||
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
|
||||
import org.apache.flink.streaming.api.functions.sink.SinkFunction;
|
||||
import org.apache.flink.streaming.api.windowing.time.Time;
|
||||
import org.awaitility.Awaitility;
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.Assert.assertArrayEquals;
|
||||
import static org.junit.Assert.assertEquals;
|
||||
|
||||
public class BackupCreatorIntegrationTest {
|
||||
public static ObjectMapper mapper;
|
||||
|
||||
@Before
|
||||
public void setup() {
|
||||
mapper = new ObjectMapper().registerModule(new JavaTimeModule());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void givenProperJson_whenDeserializeIsInvoked_thenProperObjectIsReturned() throws IOException {
|
||||
InputMessage message = new InputMessage("Me", "User", LocalDateTime.now(), "Test Message");
|
||||
byte[] messageSerialized = mapper.writeValueAsBytes(message);
|
||||
DeserializationSchema<InputMessage> deserializationSchema = new InputMessageDeserializationSchema();
|
||||
InputMessage messageDeserialized = deserializationSchema.deserialize(messageSerialized);
|
||||
|
||||
assertEquals(message, messageDeserialized);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void givenMultipleInputMessagesFromDifferentDays_whenBackupCreatorIsUser_thenMessagesAreGroupedProperly() throws Exception {
|
||||
LocalDateTime currentTime = LocalDateTime.now();
|
||||
InputMessage message = new InputMessage("Me", "User", currentTime, "First TestMessage");
|
||||
InputMessage secondMessage = new InputMessage("Me", "User", currentTime.plusHours(1), "First TestMessage");
|
||||
InputMessage thirdMessage = new InputMessage("Me", "User", currentTime.plusHours(2), "First TestMessage");
|
||||
InputMessage fourthMessage = new InputMessage("Me", "User", currentTime.plusHours(3), "First TestMessage");
|
||||
InputMessage fifthMessage = new InputMessage("Me", "User", currentTime.plusHours(25), "First TestMessage");
|
||||
InputMessage sixthMessage = new InputMessage("Me", "User", currentTime.plusHours(26), "First TestMessage");
|
||||
|
||||
List<InputMessage> firstBackupMessages = Arrays.asList(message, secondMessage, thirdMessage, fourthMessage);
|
||||
List<InputMessage> secondBackupMessages = Arrays.asList(fifthMessage, sixthMessage);
|
||||
List<InputMessage> inputMessages = ListUtils.union(firstBackupMessages, secondBackupMessages);
|
||||
|
||||
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
|
||||
env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
|
||||
env.setParallelism(1);
|
||||
DataStreamSource<InputMessage> testDataSet = env.fromCollection(inputMessages);
|
||||
CollectingSink sink = new CollectingSink();
|
||||
testDataSet.assignTimestampsAndWatermarks(new InputMessageTimestampAssigner())
|
||||
.timeWindowAll(Time.hours(24))
|
||||
.aggregate(new BackupAggregator())
|
||||
.addSink(sink);
|
||||
|
||||
env.execute();
|
||||
|
||||
Awaitility.await().until(() -> sink.backups.size() == 2);
|
||||
assertEquals(2, sink.backups.size());
|
||||
assertEquals(firstBackupMessages, sink.backups.get(0).getInputMessages());
|
||||
assertEquals(secondBackupMessages, sink.backups.get(1).getInputMessages());
|
||||
|
||||
}
|
||||
|
||||
@Test
|
||||
public void givenProperBackupObject_whenSerializeIsInvoked_thenObjectIsProperlySerialized() throws IOException {
|
||||
InputMessage message = new InputMessage("Me", "User", LocalDateTime.now(), "Test Message");
|
||||
List<InputMessage> messages = Arrays.asList(message);
|
||||
Backup backup = new Backup(messages, LocalDateTime.now());
|
||||
byte[] backupSerialized = mapper.writeValueAsBytes(backup);
|
||||
SerializationSchema<Backup> serializationSchema = new BackupSerializationSchema();
|
||||
byte[] backupProcessed = serializationSchema.serialize(backup);
|
||||
|
||||
assertArrayEquals(backupSerialized, backupProcessed);
|
||||
}
|
||||
|
||||
private static class CollectingSink implements SinkFunction<Backup> {
|
||||
|
||||
public static List<Backup> backups = new ArrayList<>();
|
||||
|
||||
@Override
|
||||
public synchronized void invoke(Backup value, Context context) throws Exception {
|
||||
backups.add(value);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
package com.baeldung.flink;
|
||||
|
||||
import com.baeldung.flink.operator.WordsCapitalizer;
|
||||
import org.apache.flink.api.java.DataSet;
|
||||
import org.apache.flink.api.java.ExecutionEnvironment;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
public class WordCapitalizerIntegrationTest {
|
||||
|
||||
@Test
|
||||
public void givenDataSet_whenExecuteWordCapitalizer_thenReturnCapitalizedWords() throws Exception {
|
||||
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
|
||||
List<String> data = Arrays.asList("dog", "cat", "wolf", "pig");
|
||||
|
||||
DataSet<String> testDataSet = env.fromCollection(data);
|
||||
|
||||
|
||||
List<String> dataProcessed = testDataSet
|
||||
.map(new WordsCapitalizer())
|
||||
.collect();
|
||||
|
||||
List<String> testDataCapitalized = data.stream()
|
||||
.map(String::toUpperCase)
|
||||
.collect(Collectors.toList());
|
||||
|
||||
Assert.assertEquals(testDataCapitalized, dataProcessed);
|
||||
}
|
||||
|
||||
}
|
||||
+100
@@ -0,0 +1,100 @@
|
||||
package com.baeldung.kafka.consumer;
|
||||
|
||||
import static org.assertj.core.api.AssertionsForInterfaceTypes.assertThat;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.kafka.clients.consumer.ConsumerRecord;
|
||||
import org.apache.kafka.clients.consumer.MockConsumer;
|
||||
import org.apache.kafka.clients.consumer.OffsetResetStrategy;
|
||||
import org.apache.kafka.common.KafkaException;
|
||||
import org.apache.kafka.common.TopicPartition;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
class CountryPopulationConsumerUnitTest {
|
||||
|
||||
private static final String TOPIC = "topic";
|
||||
private static final int PARTITION = 0;
|
||||
|
||||
private CountryPopulationConsumer countryPopulationConsumer;
|
||||
|
||||
private List<CountryPopulation> updates;
|
||||
private Throwable pollException;
|
||||
|
||||
private MockConsumer<String, Integer> consumer;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
consumer = new MockConsumer<>(OffsetResetStrategy.EARLIEST);
|
||||
updates = new ArrayList<>();
|
||||
countryPopulationConsumer = new CountryPopulationConsumer(consumer, ex -> this.pollException = ex, updates::add);
|
||||
}
|
||||
|
||||
@Test
|
||||
void whenStartingByAssigningTopicPartition_thenExpectUpdatesAreConsumedCorrectly() {
|
||||
// GIVEN
|
||||
consumer.schedulePollTask(() -> consumer.addRecord(record(TOPIC, PARTITION, "Romania", 19_410_000)));
|
||||
consumer.schedulePollTask(() -> countryPopulationConsumer.stop());
|
||||
|
||||
HashMap<TopicPartition, Long> startOffsets = new HashMap<>();
|
||||
TopicPartition tp = new TopicPartition(TOPIC, PARTITION);
|
||||
startOffsets.put(tp, 0L);
|
||||
consumer.updateBeginningOffsets(startOffsets);
|
||||
|
||||
// WHEN
|
||||
countryPopulationConsumer.startByAssigning(TOPIC, PARTITION);
|
||||
|
||||
// THEN
|
||||
assertThat(updates).hasSize(1);
|
||||
assertThat(consumer.closed()).isTrue();
|
||||
}
|
||||
|
||||
@Test
|
||||
void whenStartingBySubscribingToTopic_thenExpectUpdatesAreConsumedCorrectly() {
|
||||
// GIVEN
|
||||
consumer.schedulePollTask(() -> {
|
||||
consumer.rebalance(Collections.singletonList(new TopicPartition(TOPIC, 0)));
|
||||
consumer.addRecord(record(TOPIC, PARTITION, "Romania", 19_410_000));
|
||||
});
|
||||
consumer.schedulePollTask(() -> countryPopulationConsumer.stop());
|
||||
|
||||
HashMap<TopicPartition, Long> startOffsets = new HashMap<>();
|
||||
TopicPartition tp = new TopicPartition(TOPIC, PARTITION);
|
||||
startOffsets.put(tp, 0L);
|
||||
consumer.updateBeginningOffsets(startOffsets);
|
||||
|
||||
// WHEN
|
||||
countryPopulationConsumer.startBySubscribing(TOPIC);
|
||||
|
||||
// THEN
|
||||
assertThat(updates).hasSize(1);
|
||||
assertThat(consumer.closed()).isTrue();
|
||||
}
|
||||
|
||||
@Test
|
||||
void whenStartingBySubscribingToTopicAndExceptionOccurs_thenExpectExceptionIsHandledCorrectly() {
|
||||
// GIVEN
|
||||
consumer.schedulePollTask(() -> consumer.setPollException(new KafkaException("poll exception")));
|
||||
consumer.schedulePollTask(() -> countryPopulationConsumer.stop());
|
||||
|
||||
HashMap<TopicPartition, Long> startOffsets = new HashMap<>();
|
||||
TopicPartition tp = new TopicPartition(TOPIC, 0);
|
||||
startOffsets.put(tp, 0L);
|
||||
consumer.updateBeginningOffsets(startOffsets);
|
||||
|
||||
// WHEN
|
||||
countryPopulationConsumer.startBySubscribing(TOPIC);
|
||||
|
||||
// THEN
|
||||
assertThat(pollException).isInstanceOf(KafkaException.class).hasMessage("poll exception");
|
||||
assertThat(consumer.closed()).isTrue();
|
||||
}
|
||||
|
||||
private ConsumerRecord<String, Integer> record(String topic, int partition, String country, int population) {
|
||||
return new ConsumerRecord<>(topic, partition, 0, country, population);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
package com.baeldung.kafka.producer;
|
||||
|
||||
import com.baeldung.kafka.producer.EvenOddPartitioner;
|
||||
import com.baeldung.kafka.producer.KafkaProducer;
|
||||
import org.apache.kafka.clients.producer.MockProducer;
|
||||
import org.apache.kafka.clients.producer.RecordMetadata;
|
||||
import org.apache.kafka.common.Cluster;
|
||||
import org.apache.kafka.common.Node;
|
||||
import org.apache.kafka.common.PartitionInfo;
|
||||
import org.apache.kafka.common.serialization.StringSerializer;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.Future;
|
||||
|
||||
import static java.util.Collections.emptySet;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
class KafkaProducerUnitTest {
|
||||
|
||||
private final String TOPIC_NAME = "topic_sports_news";
|
||||
|
||||
private KafkaProducer kafkaProducer;
|
||||
private MockProducer<String, String> mockProducer;
|
||||
|
||||
private void buildMockProducer(boolean autoComplete) {
|
||||
this.mockProducer = new MockProducer<>(autoComplete, new StringSerializer(), new StringSerializer());
|
||||
}
|
||||
|
||||
@Test
|
||||
void givenKeyValue_whenSend_thenVerifyHistory() throws ExecutionException, InterruptedException {
|
||||
|
||||
buildMockProducer(true);
|
||||
//when
|
||||
kafkaProducer = new KafkaProducer(mockProducer);
|
||||
Future<RecordMetadata> recordMetadataFuture = kafkaProducer.send("data", "{\"site\" : \"baeldung\"}");
|
||||
|
||||
//then
|
||||
assertTrue(mockProducer.history().size() == 1);
|
||||
assertTrue(mockProducer.history().get(0).key().equalsIgnoreCase("data"));
|
||||
assertTrue(recordMetadataFuture.get().partition() == 0);
|
||||
|
||||
}
|
||||
|
||||
@Test
|
||||
void givenKeyValue_whenSend_thenSendOnlyAfterFlush() {
|
||||
|
||||
buildMockProducer(false);
|
||||
//when
|
||||
kafkaProducer = new KafkaProducer(mockProducer);
|
||||
Future<RecordMetadata> record = kafkaProducer.send("data", "{\"site\" : \"baeldung\"}");
|
||||
assertFalse(record.isDone());
|
||||
|
||||
//then
|
||||
kafkaProducer.flush();
|
||||
assertTrue(record.isDone());
|
||||
}
|
||||
|
||||
@Test
|
||||
void givenKeyValue_whenSend_thenReturnException() {
|
||||
|
||||
buildMockProducer(false);
|
||||
//when
|
||||
kafkaProducer = new KafkaProducer(mockProducer);
|
||||
Future<RecordMetadata> record = kafkaProducer.send("site", "{\"site\" : \"baeldung\"}");
|
||||
RuntimeException e = new RuntimeException();
|
||||
mockProducer.errorNext(e);
|
||||
//then
|
||||
try {
|
||||
record.get();
|
||||
} catch (ExecutionException | InterruptedException ex) {
|
||||
assertEquals(e, ex.getCause());
|
||||
}
|
||||
assertTrue(record.isDone());
|
||||
}
|
||||
|
||||
@Test
|
||||
void givenKeyValue_whenSendWithTxn_thenSendOnlyOnTxnCommit() {
|
||||
|
||||
buildMockProducer(true);
|
||||
//when
|
||||
kafkaProducer = new KafkaProducer(mockProducer);
|
||||
kafkaProducer.initTransaction();
|
||||
kafkaProducer.beginTransaction();
|
||||
Future<RecordMetadata> record = kafkaProducer.send("data", "{\"site\" : \"baeldung\"}");
|
||||
|
||||
//then
|
||||
assertTrue(mockProducer.history().isEmpty());
|
||||
kafkaProducer.commitTransaction();
|
||||
assertTrue(mockProducer.history().size() == 1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void givenKeyValue_whenSendWithPartitioning_thenVerifyPartitionNumber() throws ExecutionException, InterruptedException {
|
||||
|
||||
PartitionInfo partitionInfo0 = new PartitionInfo(TOPIC_NAME, 0, null, null, null);
|
||||
PartitionInfo partitionInfo1 = new PartitionInfo(TOPIC_NAME, 1, null, null, null);
|
||||
List<PartitionInfo> list = new ArrayList<>();
|
||||
list.add(partitionInfo0);
|
||||
list.add(partitionInfo1);
|
||||
Cluster cluster = new Cluster("kafkab", new ArrayList<Node>(), list, emptySet(), emptySet());
|
||||
this.mockProducer = new MockProducer<>(cluster, true, new EvenOddPartitioner(), new StringSerializer(), new StringSerializer());
|
||||
//when
|
||||
kafkaProducer = new KafkaProducer(mockProducer);
|
||||
Future<RecordMetadata> recordMetadataFuture = kafkaProducer.send("partition", "{\"site\" : \"baeldung\"}");
|
||||
|
||||
//then
|
||||
assertTrue(recordMetadataFuture.get().partition() == 1);
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
Reference in New Issue
Block a user