From cb6e84c59e505f20988e2fee91fde1442fc6077e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Norberto=20Ritzmann=20J=C3=BAnior?= Date: Sat, 6 Oct 2018 00:09:13 -0300 Subject: [PATCH] New Article - ETL with Spring Cloud Data Flow (#5388) * Including the ETL files to the repository * Rename READMET.md to README.MD Renamed to correct README. * Update README.MD Additional line brakes * Formatting * Removing unecessary autogenerated files --- spring-cloud-data-flow/etl/README.MD | 9 +++ .../etl/customer-mongodb-sink/pom.xml | 75 +++++++++++++++++++ .../customermongodbsink/Customer.java | 27 +++++++ .../customermongodbsink/CustomerListener.java | 18 +++++ .../CustomerMongodbSinkApplication.java | 12 +++ .../CustomerRepository.java | 9 +++ .../src/main/resources/application.properties | 0 .../etl/customer-transform/pom.xml | 68 +++++++++++++++++ .../customer/customertransform/Customer.java | 29 +++++++ .../CustomerProcessorConfiguration.java | 16 ++++ .../CustomerTransformApplication.java | 12 +++ .../src/main/resources/application.properties | 0 spring-cloud-data-flow/etl/pom.xml | 20 +++++ spring-cloud-data-flow/pom.xml | 1 + 14 files changed, 296 insertions(+) create mode 100644 spring-cloud-data-flow/etl/README.MD create mode 100644 spring-cloud-data-flow/etl/customer-mongodb-sink/pom.xml create mode 100644 spring-cloud-data-flow/etl/customer-mongodb-sink/src/main/java/com/customer/customermongodbsink/Customer.java create mode 100644 spring-cloud-data-flow/etl/customer-mongodb-sink/src/main/java/com/customer/customermongodbsink/CustomerListener.java create mode 100644 spring-cloud-data-flow/etl/customer-mongodb-sink/src/main/java/com/customer/customermongodbsink/CustomerMongodbSinkApplication.java create mode 100644 spring-cloud-data-flow/etl/customer-mongodb-sink/src/main/java/com/customer/customermongodbsink/CustomerRepository.java create mode 100644 spring-cloud-data-flow/etl/customer-mongodb-sink/src/main/resources/application.properties create mode 100644 spring-cloud-data-flow/etl/customer-transform/pom.xml create mode 100644 spring-cloud-data-flow/etl/customer-transform/src/main/java/com/customer/customertransform/Customer.java create mode 100644 spring-cloud-data-flow/etl/customer-transform/src/main/java/com/customer/customertransform/CustomerProcessorConfiguration.java create mode 100644 spring-cloud-data-flow/etl/customer-transform/src/main/java/com/customer/customertransform/CustomerTransformApplication.java create mode 100644 spring-cloud-data-flow/etl/customer-transform/src/main/resources/application.properties create mode 100644 spring-cloud-data-flow/etl/pom.xml diff --git a/spring-cloud-data-flow/etl/README.MD b/spring-cloud-data-flow/etl/README.MD new file mode 100644 index 0000000000..0cbb460b01 --- /dev/null +++ b/spring-cloud-data-flow/etl/README.MD @@ -0,0 +1,9 @@ +# Overview +This is an example of a ETL stream pipeline, mixing a starter application with custom transform and sink. + +# Applications +JDBC Source - Application Starter distributed by default + +customer-transform - Custom application to transform the data + +customer-mongodb-sink - Custom application to sink the data diff --git a/spring-cloud-data-flow/etl/customer-mongodb-sink/pom.xml b/spring-cloud-data-flow/etl/customer-mongodb-sink/pom.xml new file mode 100644 index 0000000000..468d8e17d0 --- /dev/null +++ b/spring-cloud-data-flow/etl/customer-mongodb-sink/pom.xml @@ -0,0 +1,75 @@ + + + 4.0.0 + + com.customer + customer-mongodb-sink + jar + + customer-mongodb-sink + Example ETL Load Project + + + parent-boot-2 + com.baeldung + 0.0.1-SNAPSHOT + ../../../parent-boot-2 + + + + UTF-8 + UTF-8 + 1.8 + Finchley.SR1 + + + + + org.springframework.cloud + spring-cloud-stream + + + org.springframework.cloud + spring-cloud-stream-binder-rabbit + + + org.springframework.boot + spring-boot-starter-data-mongodb + + + org.springframework.boot + spring-boot-starter-test + test + + + org.springframework.cloud + spring-cloud-stream-test-support + test + + + + + + + org.springframework.cloud + spring-cloud-dependencies + ${spring-cloud.version} + pom + import + + + + + + + + org.springframework.boot + spring-boot-maven-plugin + + + + + + diff --git a/spring-cloud-data-flow/etl/customer-mongodb-sink/src/main/java/com/customer/customermongodbsink/Customer.java b/spring-cloud-data-flow/etl/customer-mongodb-sink/src/main/java/com/customer/customermongodbsink/Customer.java new file mode 100644 index 0000000000..cf44aec5b7 --- /dev/null +++ b/spring-cloud-data-flow/etl/customer-mongodb-sink/src/main/java/com/customer/customermongodbsink/Customer.java @@ -0,0 +1,27 @@ +package com.customer.customermongodbsink; + +import org.springframework.data.mongodb.core.mapping.Document; + +@Document(collection = "customer") +public class Customer { + + private Long id; + private String name; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + +} \ No newline at end of file diff --git a/spring-cloud-data-flow/etl/customer-mongodb-sink/src/main/java/com/customer/customermongodbsink/CustomerListener.java b/spring-cloud-data-flow/etl/customer-mongodb-sink/src/main/java/com/customer/customermongodbsink/CustomerListener.java new file mode 100644 index 0000000000..c841daea8a --- /dev/null +++ b/spring-cloud-data-flow/etl/customer-mongodb-sink/src/main/java/com/customer/customermongodbsink/CustomerListener.java @@ -0,0 +1,18 @@ +package com.customer.customermongodbsink; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.cloud.stream.annotation.EnableBinding; +import org.springframework.cloud.stream.annotation.StreamListener; +import org.springframework.cloud.stream.messaging.Sink; + +@EnableBinding(Sink.class) +public class CustomerListener { + + @Autowired + private CustomerRepository repository; + + @StreamListener(Sink.INPUT) + public void save(Customer customer) { + repository.save(customer); + } +} diff --git a/spring-cloud-data-flow/etl/customer-mongodb-sink/src/main/java/com/customer/customermongodbsink/CustomerMongodbSinkApplication.java b/spring-cloud-data-flow/etl/customer-mongodb-sink/src/main/java/com/customer/customermongodbsink/CustomerMongodbSinkApplication.java new file mode 100644 index 0000000000..2ef311457e --- /dev/null +++ b/spring-cloud-data-flow/etl/customer-mongodb-sink/src/main/java/com/customer/customermongodbsink/CustomerMongodbSinkApplication.java @@ -0,0 +1,12 @@ +package com.customer.customermongodbsink; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; + +@SpringBootApplication +public class CustomerMongodbSinkApplication { + + public static void main(String[] args) { + SpringApplication.run(CustomerMongodbSinkApplication.class, args); + } +} diff --git a/spring-cloud-data-flow/etl/customer-mongodb-sink/src/main/java/com/customer/customermongodbsink/CustomerRepository.java b/spring-cloud-data-flow/etl/customer-mongodb-sink/src/main/java/com/customer/customermongodbsink/CustomerRepository.java new file mode 100644 index 0000000000..f921ff51cf --- /dev/null +++ b/spring-cloud-data-flow/etl/customer-mongodb-sink/src/main/java/com/customer/customermongodbsink/CustomerRepository.java @@ -0,0 +1,9 @@ +package com.customer.customermongodbsink; + +import org.springframework.data.mongodb.repository.MongoRepository; +import org.springframework.stereotype.Repository; + +@Repository +public interface CustomerRepository extends MongoRepository { + +} \ No newline at end of file diff --git a/spring-cloud-data-flow/etl/customer-mongodb-sink/src/main/resources/application.properties b/spring-cloud-data-flow/etl/customer-mongodb-sink/src/main/resources/application.properties new file mode 100644 index 0000000000..e69de29bb2 diff --git a/spring-cloud-data-flow/etl/customer-transform/pom.xml b/spring-cloud-data-flow/etl/customer-transform/pom.xml new file mode 100644 index 0000000000..bc4b648907 --- /dev/null +++ b/spring-cloud-data-flow/etl/customer-transform/pom.xml @@ -0,0 +1,68 @@ + + + 4.0.0 + + com.customer + customer-transform + 0.0.1-SNAPSHOT + jar + + customer-transform + Example transform ETL step + + + parent-boot-2 + com.baeldung + 0.0.1-SNAPSHOT + ../../../parent-boot-2 + + + + UTF-8 + UTF-8 + 1.8 + Finchley.SR1 + + + + + org.springframework.cloud + spring-cloud-stream-binder-rabbit + + + org.springframework.boot + spring-boot-starter-test + test + + + org.springframework.cloud + spring-cloud-stream-test-support + test + + + + + + + org.springframework.cloud + spring-cloud-dependencies + ${spring-cloud.version} + pom + import + + + + + + + + org.springframework.boot + spring-boot-maven-plugin + + + + + + diff --git a/spring-cloud-data-flow/etl/customer-transform/src/main/java/com/customer/customertransform/Customer.java b/spring-cloud-data-flow/etl/customer-transform/src/main/java/com/customer/customertransform/Customer.java new file mode 100644 index 0000000000..f0e4d79388 --- /dev/null +++ b/spring-cloud-data-flow/etl/customer-transform/src/main/java/com/customer/customertransform/Customer.java @@ -0,0 +1,29 @@ +package com.customer.customertransform; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public class Customer { + + private Long id; + + private String name; + + @JsonProperty("customer_name") + public void setName(String name) { + this.name = name; + } + + @JsonProperty("name") + public String getName() { + return name; + } + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + +} \ No newline at end of file diff --git a/spring-cloud-data-flow/etl/customer-transform/src/main/java/com/customer/customertransform/CustomerProcessorConfiguration.java b/spring-cloud-data-flow/etl/customer-transform/src/main/java/com/customer/customertransform/CustomerProcessorConfiguration.java new file mode 100644 index 0000000000..c99fcf55be --- /dev/null +++ b/spring-cloud-data-flow/etl/customer-transform/src/main/java/com/customer/customertransform/CustomerProcessorConfiguration.java @@ -0,0 +1,16 @@ +package com.customer.customertransform; + + +import org.springframework.cloud.stream.annotation.EnableBinding; +import org.springframework.cloud.stream.messaging.Processor; +import org.springframework.integration.annotation.Transformer; + +@EnableBinding(Processor.class) +public class CustomerProcessorConfiguration { + + @Transformer(inputChannel = Processor.INPUT, outputChannel = Processor.OUTPUT) + public Customer convertToPojo(Customer payload) { + + return payload; + } +} \ No newline at end of file diff --git a/spring-cloud-data-flow/etl/customer-transform/src/main/java/com/customer/customertransform/CustomerTransformApplication.java b/spring-cloud-data-flow/etl/customer-transform/src/main/java/com/customer/customertransform/CustomerTransformApplication.java new file mode 100644 index 0000000000..8781f4da54 --- /dev/null +++ b/spring-cloud-data-flow/etl/customer-transform/src/main/java/com/customer/customertransform/CustomerTransformApplication.java @@ -0,0 +1,12 @@ +package com.customer.customertransform; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; + +@SpringBootApplication +public class CustomerTransformApplication { + + public static void main(String[] args) { + SpringApplication.run(CustomerTransformApplication.class, args); + } +} diff --git a/spring-cloud-data-flow/etl/customer-transform/src/main/resources/application.properties b/spring-cloud-data-flow/etl/customer-transform/src/main/resources/application.properties new file mode 100644 index 0000000000..e69de29bb2 diff --git a/spring-cloud-data-flow/etl/pom.xml b/spring-cloud-data-flow/etl/pom.xml new file mode 100644 index 0000000000..2b904f6e0d --- /dev/null +++ b/spring-cloud-data-flow/etl/pom.xml @@ -0,0 +1,20 @@ + + 4.0.0 + org.baeldung.spring.cloud + etl-spring-cloud-data-flow + 0.0.1-SNAPSHOT + pom + + + org.baeldung.spring.cloud + spring-cloud-data-flow + 0.0.1-SNAPSHOT + + + + customer-mongodb-sink + customer-transform + + + diff --git a/spring-cloud-data-flow/pom.xml b/spring-cloud-data-flow/pom.xml index 5f24aa2cbd..5a007f3c7d 100644 --- a/spring-cloud-data-flow/pom.xml +++ b/spring-cloud-data-flow/pom.xml @@ -19,6 +19,7 @@ time-processor log-sink batch-job + etl