diff --git a/spring-cloud/README.md b/spring-cloud/README.md index 53b7606563..adaf3052e6 100644 --- a/spring-cloud/README.md +++ b/spring-cloud/README.md @@ -12,13 +12,11 @@ Client-side service discovery allows services to find and communicate with each other without hardcoding hostname and port. The only ‘fixed point’ in such an architecture consists of a service registry with which each service has to register. +### Relevant Articles: - [Intro to Spring Cloud Netflix - Hystrix](http://www.baeldung.com/spring-cloud-netflix-hystrix) - [Dockerizing a Spring Boot Application](http://www.baeldung.com/dockerizing-spring-boot-application) - [Introduction to Spring Cloud Rest Client with Netflix Ribbon](http://www.baeldung.com/spring-cloud-rest-client-with-netflix-ribbon) - [A Quick Guide to Spring Cloud Consul](http://www.baeldung.com/spring-cloud-consul) - -### Relevant Articles: -- [Introduction to Spring Cloud Rest Client with Netflix Ribbon](http://www.baeldung.com/spring-cloud-rest-client-with-netflix-ribbon) - [An Introduction to Spring Cloud Zookeeper](http://www.baeldung.com/spring-cloud-zookeeper) +- [Using a Spring Cloud App Starter](http://www.baeldung.com/using-a-spring-cloud-app-starter) - [Spring Cloud Connectors and Heroku](http://www.baeldung.com/spring-cloud-heroku) - diff --git a/spring-cloud/spring-cloud-stream-starters/bash/hadoop.sh b/spring-cloud/spring-cloud-stream-starters/bash/hadoop.sh new file mode 100644 index 0000000000..ca8298430b --- /dev/null +++ b/spring-cloud/spring-cloud-stream-starters/bash/hadoop.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash + +# For Ubuntu 14.04 +# Inspired from: https://github.com/curran/setupHadoop/blob/master/setupHadoop.sh +# Use from the user directory + +# Install Java +sudo apt-get update +sudo add-apt-repository -y ppa:webupd8team/java +sudo apt-get install -y oracle-java8-installer + +# Install Hadoop +curl -O http://mirror.cogentco.com/pub/apache/hadoop/common/hadoop-2.8.2/hadoop-2.8.2.tar.gz +tar xfz hadoop-2.8.2.tar.gz +sudo mv hadoop-2.8.2 /usr/local/hadoop +rm hadoop-2.8.2.tar.gz + +# Environmental Variables +echo export JAVA_HOME=/usr/lib/jvm/java-8-oracle >> ~/.bashrc +echo export HADOOP_PREFIX=/usr/local/hadoop >> ~/.bashrc +echo export PATH=\$PATH:/usr/local/hadoop/bin >> ~/.bashrc +echo export PATH=\$PATH:/usr/local/hadoop/sbin >> ~/.bashrc +source ~/.bashrc + +# Copy configuration files +cp master/* /usr/local/hadoop/etc/hadoop/ + +# Format HDFS +hdfs namenode -format + +# SSH keys for Hadoop to use. +ssh-keygen -t rsa -P 'password' -f ~/.ssh/id_rsa.pub +sudo mv ~/.ssh/id_rsa.pub ~/.ssh/authorized_keys + +# SSH +ssh localhost +# authenticate with osboxes.org + +# Start NameNode daemon and DataNode daemon +start-dfs.sh +# stop-dfs.sh + +# Install Maven +sudo apt-get install maven + +# Access Hadoop - http://localhost:50070 \ No newline at end of file diff --git a/spring-cloud/spring-cloud-stream-starters/boot/.gitignore b/spring-cloud/spring-cloud-stream-starters/boot/.gitignore new file mode 100644 index 0000000000..e4b82e1c0f --- /dev/null +++ b/spring-cloud/spring-cloud-stream-starters/boot/.gitignore @@ -0,0 +1,3 @@ +.idea +*/target/* +*.iml diff --git a/spring-cloud/spring-cloud-stream-starters/boot/pom.xml b/spring-cloud/spring-cloud-stream-starters/boot/pom.xml new file mode 100644 index 0000000000..3e6bc134e3 --- /dev/null +++ b/spring-cloud/spring-cloud-stream-starters/boot/pom.xml @@ -0,0 +1,55 @@ + + + 4.0.0 + com.baeldung.twitterhdfs + twitterhdfs + jar + 1.0.0 + + twitterhdfs + + + UTF-8 + UTF-8 + 1.8 + + + + org.springframework.boot + spring-boot-starter-parent + 1.5.8.RELEASE + + + + + + org.springframework.cloud.stream.app + spring-cloud-starter-stream-source-twitterstream + 1.3.1.RELEASE + + + org.springframework.cloud.stream.app + spring-cloud-starter-stream-sink-hdfs + 1.3.1.RELEASE + + + + + javax.servlet + jstl + + + + + + + + org.springframework.boot + spring-boot-maven-plugin + + + twitterhdfs + + \ No newline at end of file diff --git a/spring-cloud/spring-cloud-stream-starters/boot/src/main/java/com/baeldung/twitterhdfs/aggregate/AggregateApp.java b/spring-cloud/spring-cloud-stream-starters/boot/src/main/java/com/baeldung/twitterhdfs/aggregate/AggregateApp.java new file mode 100644 index 0000000000..8b9ca6dc62 --- /dev/null +++ b/spring-cloud/spring-cloud-stream-starters/boot/src/main/java/com/baeldung/twitterhdfs/aggregate/AggregateApp.java @@ -0,0 +1,18 @@ +package com.baeldung.twitterhdfs.aggregate; + +import com.baeldung.twitterhdfs.processor.ProcessorApp; +import com.baeldung.twitterhdfs.source.SourceApp; +import com.baeldung.twitterhdfs.sink.SinkApp; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.cloud.stream.aggregate.AggregateApplicationBuilder; + +@SpringBootApplication +public class AggregateApp { + public static void main(String[] args) { + new AggregateApplicationBuilder() + .from(SourceApp.class).args("--fixedDelay=5000") + .via(ProcessorApp.class) + .to(SinkApp.class).args("--debug=true") + .run(args); + } +} \ No newline at end of file diff --git a/spring-cloud/spring-cloud-stream-starters/boot/src/main/java/com/baeldung/twitterhdfs/processor/ProcessorApp.java b/spring-cloud/spring-cloud-stream-starters/boot/src/main/java/com/baeldung/twitterhdfs/processor/ProcessorApp.java new file mode 100644 index 0000000000..e3bd1197f6 --- /dev/null +++ b/spring-cloud/spring-cloud-stream-starters/boot/src/main/java/com/baeldung/twitterhdfs/processor/ProcessorApp.java @@ -0,0 +1,20 @@ +package com.baeldung.twitterhdfs.processor; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.cloud.stream.annotation.EnableBinding; +import org.springframework.cloud.stream.messaging.Processor; +import org.springframework.integration.annotation.Transformer; + +@SpringBootApplication +@EnableBinding(Processor.class) +public class ProcessorApp { + Logger log = LoggerFactory.getLogger(ProcessorApp.class); + + @Transformer(inputChannel = Processor.INPUT, outputChannel = Processor.OUTPUT) + public String processMessage(String payload) { + log.info("Payload received!"); + return payload; + } +} \ No newline at end of file diff --git a/spring-cloud/spring-cloud-stream-starters/boot/src/main/java/com/baeldung/twitterhdfs/sink/SinkApp.java b/spring-cloud/spring-cloud-stream-starters/boot/src/main/java/com/baeldung/twitterhdfs/sink/SinkApp.java new file mode 100644 index 0000000000..c0c1e287d3 --- /dev/null +++ b/spring-cloud/spring-cloud-stream-starters/boot/src/main/java/com/baeldung/twitterhdfs/sink/SinkApp.java @@ -0,0 +1,22 @@ +package com.baeldung.twitterhdfs.sink; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.cloud.stream.annotation.EnableBinding; +import org.springframework.cloud.stream.app.hdfs.sink.HdfsSinkConfiguration; +import org.springframework.cloud.stream.messaging.Sink; +import org.springframework.context.annotation.Import; +import org.springframework.integration.annotation.ServiceActivator; + +@SpringBootApplication +@EnableBinding(Sink.class) +@Import(HdfsSinkConfiguration.class) +public class SinkApp { + Logger log = LoggerFactory.getLogger(SinkApp.class); + + @ServiceActivator(inputChannel= Sink.INPUT) + public void loggerSink(Object payload) { + log.info("Received: " + payload); + } +} \ No newline at end of file diff --git a/spring-cloud/spring-cloud-stream-starters/boot/src/main/java/com/baeldung/twitterhdfs/source/SourceApp.java b/spring-cloud/spring-cloud-stream-starters/boot/src/main/java/com/baeldung/twitterhdfs/source/SourceApp.java new file mode 100644 index 0000000000..f9b220561b --- /dev/null +++ b/spring-cloud/spring-cloud-stream-starters/boot/src/main/java/com/baeldung/twitterhdfs/source/SourceApp.java @@ -0,0 +1,26 @@ +package com.baeldung.twitterhdfs.source; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.cloud.stream.annotation.EnableBinding; +import org.springframework.cloud.stream.app.twitterstream.source.TwitterstreamSourceConfiguration; +import org.springframework.cloud.stream.messaging.Source; +import org.springframework.context.annotation.Import; +import org.springframework.integration.annotation.InboundChannelAdapter; + +import java.text.SimpleDateFormat; +import java.util.Date; + +@SpringBootApplication +@EnableBinding(Source.class) +@Import(TwitterstreamSourceConfiguration.class) +public class SourceApp { + Logger log = LoggerFactory.getLogger(SourceApp.class); + + @InboundChannelAdapter(value = Source.OUTPUT) + public String timerMessageSource() { + return new SimpleDateFormat().format(new Date()); + } + +} diff --git a/spring-cloud/spring-cloud-stream-starters/boot/src/main/resources/application.properties b/spring-cloud/spring-cloud-stream-starters/boot/src/main/resources/application.properties new file mode 100644 index 0000000000..298a8ebf4d --- /dev/null +++ b/spring-cloud/spring-cloud-stream-starters/boot/src/main/resources/application.properties @@ -0,0 +1,6 @@ +hdfs.fs-uri=hdfs://127.0.0.1:50010/ + +twitter.credentials.access-token= +twitter.credentials.access-token-secret= +twitter.credentials.consumer-key= +twitter.credentials.consumer-secret= \ No newline at end of file diff --git a/spring-cloud/spring-cloud-stream-starters/hdfs/application.properties b/spring-cloud/spring-cloud-stream-starters/hdfs/application.properties new file mode 100644 index 0000000000..1f4aaf88dd --- /dev/null +++ b/spring-cloud/spring-cloud-stream-starters/hdfs/application.properties @@ -0,0 +1 @@ +hdfs.fs-uri=hdfs://127.0.0.1:50010/ \ No newline at end of file diff --git a/spring-cloud/spring-cloud-stream-starters/hdfs/hdfs.sh b/spring-cloud/spring-cloud-stream-starters/hdfs/hdfs.sh new file mode 100644 index 0000000000..577a25dd6e --- /dev/null +++ b/spring-cloud/spring-cloud-stream-starters/hdfs/hdfs.sh @@ -0,0 +1,11 @@ +# Git spring-cloud-stream-app-starters +# https://github.com/spring-cloud-stream-app-starters/hdfs/blob/master/spring-cloud-starter-stream-sink-hdfs/README.adoc +git clone https://github.com/spring-cloud-stream-app-starters/hdfs.git + +# Build it +./mvnw clean install -PgenerateApps + +# Run it +cd apps +# Optionally inject application.properties prior to build +java -jar hdfs-sink.jar --fsUri=hdfs://127.0.0.1:50010/ \ No newline at end of file diff --git a/spring-cloud/spring-cloud-stream-starters/twitter/application.properties b/spring-cloud/spring-cloud-stream-starters/twitter/application.properties new file mode 100644 index 0000000000..e38612d25e --- /dev/null +++ b/spring-cloud/spring-cloud-stream-starters/twitter/application.properties @@ -0,0 +1,4 @@ +twitter.credentials.access-token= +twitter.credentials.access-token-secret= +twitter.credentials.consumer-key= +twitter.credentials.consumer-secret= \ No newline at end of file diff --git a/spring-cloud/spring-cloud-stream-starters/twitter/twitter.sh b/spring-cloud/spring-cloud-stream-starters/twitter/twitter.sh new file mode 100644 index 0000000000..4c76fe637b --- /dev/null +++ b/spring-cloud/spring-cloud-stream-starters/twitter/twitter.sh @@ -0,0 +1,12 @@ +# Git spring-cloud-stream-app-starters +# https://github.com/spring-cloud-stream-app-starters/hdfs/blob/master/spring-cloud-starter-stream-sink-hdfs/README.adoc +git clone https://github.com/spring-cloud-stream-app-starters/twitter.git + +# Build it +./mvnw clean install -PgenerateApps + +# Run it +cd apps +# Optionally inject application.properties prior to build +java -jar twitter_stream_source.jar --consumerKey= --consumerSecret= \ + --accessToken= --accessTokenSecret= \ No newline at end of file