Merge branch 'master' into master

This commit is contained in:
Eric Martin
2019-06-07 19:02:45 -05:00
committed by GitHub
68 changed files with 1845 additions and 197 deletions
@@ -0,0 +1,40 @@
package com.baeldung.xml
import groovy.xml.MarkupBuilder
import groovy.xml.XmlUtil
import spock.lang.Specification
class MarkupBuilderUnitTest extends Specification {
def xmlFile = getClass().getResource("articles_short_formatted.xml")
def "Should create XML properly"() {
given: "Node structures"
when: "Using MarkupBuilderUnitTest to create com.baeldung.xml structure"
def writer = new StringWriter()
new MarkupBuilder(writer).articles {
article {
title('First steps in Java')
author(id: '1') {
firstname('Siena')
lastname('Kerr')
}
'release-date'('2018-12-01')
}
article {
title('Dockerize your SpringBoot application')
author(id: '2') {
firstname('Jonas')
lastname('Lugo')
}
'release-date'('2018-12-01')
}
}
then: "Xml is created properly"
XmlUtil.serialize(writer.toString()) == XmlUtil.serialize(xmlFile.text)
}
}
@@ -0,0 +1,94 @@
package com.baeldung.xml
import spock.lang.Shared
import spock.lang.Specification
class XmlParserUnitTest extends Specification {
def xmlFile = getClass().getResourceAsStream("articles.xml")
@Shared
def parser = new XmlParser()
def "Should read XML file properly"() {
given: "XML file"
when: "Using XmlParser to read file"
def articles = parser.parse(xmlFile)
then: "Xml is loaded properly"
articles.'*'.size() == 4
articles.article[0].author.firstname.text() == "Siena"
articles.article[2].'release-date'.text() == "2018-06-12"
articles.article[3].title.text() == "Java 12 insights"
articles.article.find { it.author.'@id'.text() == "3" }.author.firstname.text() == "Daniele"
}
def "Should add node to existing com.baeldung.xml using NodeBuilder"() {
given: "XML object"
def articles = parser.parse(xmlFile)
when: "Adding node to com.baeldung.xml"
def articleNode = new NodeBuilder().article(id: '5') {
title('Traversing XML in the nutshell')
author {
firstname('Martin')
lastname('Schmidt')
}
'release-date'('2019-05-18')
}
articles.append(articleNode)
then: "Node is added to com.baeldung.xml properly"
articles.'*'.size() == 5
articles.article[4].title.text() == "Traversing XML in the nutshell"
}
def "Should replace node"() {
given: "XML object"
def articles = parser.parse(xmlFile)
when: "Adding node to com.baeldung.xml"
def articleNode = new NodeBuilder().article(id: '5') {
title('Traversing XML in the nutshell')
author {
firstname('Martin')
lastname('Schmidt')
}
'release-date'('2019-05-18')
}
articles.article[0].replaceNode(articleNode)
then: "Node is added to com.baeldung.xml properly"
articles.'*'.size() == 4
articles.article[0].title.text() == "Traversing XML in the nutshell"
}
def "Should modify node"() {
given: "XML object"
def articles = parser.parse(xmlFile)
when: "Changing value of one of the nodes"
articles.article.each { it.'release-date'[0].value = "2019-05-18" }
then: "XML is updated"
articles.article.findAll { it.'release-date'.text() != "2019-05-18" }.isEmpty()
}
def "Should remove article from com.baeldung.xml"() {
given: "XML object"
def articles = parser.parse(xmlFile)
when: "Removing all articles but with id==3"
articles.article
.findAll { it.author.'@id'.text() != "3" }
.each { articles.remove(it) }
then: "There is only one article left"
articles.children().size() == 1
articles.article[0].author.'@id'.text() == "3"
}
}
@@ -0,0 +1,102 @@
package com.baeldung.xml
import groovy.xml.XmlUtil
import spock.lang.Shared
import spock.lang.Specification
class XmlSlurperUnitTest extends Specification {
def xmlFile = getClass().getResourceAsStream("articles.xml")
@Shared
def parser = new XmlSlurper()
def "Should read XML file properly"() {
given: "XML file"
when: "Using XmlSlurper to read file"
def articles = parser.parse(xmlFile)
then: "Xml is loaded properly"
articles.'*'.size() == 4
articles.article[0].author.firstname == "Siena"
articles.article[2].'release-date' == "2018-06-12"
articles.article[3].title == "Java 12 insights"
articles.article.find { it.author.'@id' == "3" }.author.firstname == "Daniele"
}
def "Should add node to existing com.baeldung.xml"() {
given: "XML object"
def articles = parser.parse(xmlFile)
when: "Adding node to com.baeldung.xml"
articles.appendNode {
article(id: '5') {
title('Traversing XML in the nutshell')
author {
firstname('Martin')
lastname('Schmidt')
}
'release-date'('2019-05-18')
}
}
articles = parser.parseText(XmlUtil.serialize(articles))
then: "Node is added to com.baeldung.xml properly"
articles.'*'.size() == 5
articles.article[4].title == "Traversing XML in the nutshell"
}
def "Should modify node"() {
given: "XML object"
def articles = parser.parse(xmlFile)
when: "Changing value of one of the nodes"
articles.article.each { it.'release-date' = "2019-05-18" }
then: "XML is updated"
articles.article.findAll { it.'release-date' != "2019-05-18" }.isEmpty()
}
def "Should replace node"() {
given: "XML object"
def articles = parser.parse(xmlFile)
when: "Replacing node"
articles.article[0].replaceNode {
article(id: '5') {
title('Traversing XML in the nutshell')
author {
firstname('Martin')
lastname('Schmidt')
}
'release-date'('2019-05-18')
}
}
articles = parser.parseText(XmlUtil.serialize(articles))
then: "Node is replaced properly"
articles.'*'.size() == 4
articles.article[0].title == "Traversing XML in the nutshell"
}
def "Should remove article from com.baeldung.xml"() {
given: "XML object"
def articles = parser.parse(xmlFile)
when: "Removing all articles but with id==3"
articles.article
.findAll { it.author.'@id' != "3" }
.replaceNode {}
articles = parser.parseText(XmlUtil.serialize(articles))
then: "There is only one article left"
articles.children().size() == 1
articles.article[0].author.'@id' == "3"
}
}
@@ -0,0 +1,34 @@
<articles>
<article>
<title>First steps in Java</title>
<author id='1'>
<firstname>Siena</firstname>
<lastname>Kerr</lastname>
</author>
<release-date>2018-12-01</release-date>
</article>
<article>
<title>Dockerize your SpringBoot application</title>
<author id='2'>
<firstname>Jonas</firstname>
<lastname>Lugo</lastname>
</author>
<release-date>2018-12-01</release-date>
</article>
<article>
<title>SpringBoot tutorial</title>
<author id='3'>
<firstname>Daniele</firstname>
<lastname>Ferguson</lastname>
</author>
<release-date>2018-06-12</release-date>
</article>
<article>
<title>Java 12 insights</title>
<author id='1'>
<firstname>Siena</firstname>
<lastname>Kerr</lastname>
</author>
<release-date>2018-07-22</release-date>
</article>
</articles>
@@ -0,0 +1,18 @@
<articles>
<article>
<title>First steps in Java</title>
<author id='1'>
<firstname>Siena</firstname>
<lastname>Kerr</lastname>
</author>
<release-date>2018-12-01</release-date>
</article>
<article>
<title>Dockerize your SpringBoot application</title>
<author id='2'>
<firstname>Jonas</firstname>
<lastname>Lugo</lastname>
</author>
<release-date>2018-12-01</release-date>
</article>
</articles>
@@ -1,15 +0,0 @@
package com.baeldung.error;
public class ErrorGenerator {
public void throwException() throws Exception {
throw new Exception("checked");
}
public void throwRuntimeException() {
throw new RuntimeException("unchecked");
}
public void throwError() {
throw new Error("unchecked");
}
}
@@ -1,51 +0,0 @@
package com.baeldung.error;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
public class ErrorGeneratorUnitTest {
private ErrorGenerator errorGenerator;
@Before
public void setUp() {
errorGenerator = new ErrorGenerator();
}
@Test
public void whenCheckedException_thenIsCaughtByCatchException() {
try {
errorGenerator.throwException();
} catch (Exception e) {
// caught! -> test pass
}
}
@Test
public void whenUncheckedException_thenIsCaughtByCatchException() {
try {
errorGenerator.throwRuntimeException();
} catch (Exception e) {
// caught! -> test pass
}
}
@Test(expected = Error.class)
public void whenError_thenIsNotCaughtByCatchException() {
try {
errorGenerator.throwError();
} catch (Exception e) {
Assert.fail(); // errors are not caught by catch exception
}
}
@Test
public void whenError_thenIsCaughtByCatchError() {
try {
errorGenerator.throwError();
} catch (Error e) {
// caught! -> test pass
}
}
}
+18 -1
View File
@@ -22,6 +22,9 @@
<maven.compiler.source>1.8</maven.compiler.source>
<maven.compiler.target>1.8</maven.compiler.target>
<icu.version>64.2</icu.version>
<hibernate.core.version>5.4.0.Final</hibernate.core.version>
<h2database.version>1.4.197</h2database.version>
<jackson.databind.version>2.9.8</jackson.databind.version>
</properties>
<dependencies>
@@ -30,7 +33,21 @@
<artifactId>icu4j</artifactId>
<version>${icu.version}</version>
</dependency>
<dependency>
<groupId>org.hibernate</groupId>
<artifactId>hibernate-core</artifactId>
<version>${hibernate.core.version}</version>
</dependency>
<dependency>
<groupId>com.h2database</groupId>
<artifactId>h2</artifactId>
<version>${h2database.version}</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>${jackson.databind.version}</version>
</dependency>
</dependencies>
<build>
@@ -0,0 +1,41 @@
package com.baeldung.optionalReturnType;
import java.util.HashMap;
import java.util.Map;
import java.util.Optional;
public class HandleOptionalTypeExample {
static Map<String, User> usersByName = new HashMap();
static {
User user1 = new User();
user1.setUserId(1l);
user1.setFirstName("baeldung");
usersByName.put("baeldung", user1);
}
public static void main(String[] args) {
changeUserName("baeldung", "baeldung-new");
changeUserName("user", "user-new");
}
public static void changeUserName(String oldFirstName, String newFirstName) {
Optional<User> userOpt = findUserByName(oldFirstName);
if (userOpt.isPresent()) {
User user = userOpt.get();
user.setFirstName(newFirstName);
System.out.println("user with name " + oldFirstName + " is changed to " + user.getFirstName());
} else {
// user is missing
System.out.println("user with name " + oldFirstName + " is not found.");
}
}
public static Optional<User> findUserByName(String name) {
// look up the user in the database, the user object below could be null
User user = usersByName.get(name);
Optional<User> opt = Optional.ofNullable(user);
return opt;
}
}
@@ -0,0 +1,19 @@
package com.baeldung.optionalReturnType;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
public class OptionalToJsonExample {
public static void main(String[] args) {
UserOptional user = new UserOptional();
user.setUserId(1l);
user.setFirstName("Bael Dung");
ObjectMapper om = new ObjectMapper();
try {
System.out.print("user in json is:" + om.writeValueAsString(user));
} catch (JsonProcessingException e) {
e.printStackTrace();
}
}
}
@@ -0,0 +1,26 @@
package com.baeldung.optionalReturnType;
import java.util.Optional;
import javax.persistence.EntityManager;
import javax.persistence.EntityManagerFactory;
import javax.persistence.Persistence;
public class PersistOptionalTypeExample {
static String persistenceUnit = "com.baeldung.optionalReturnType";
static EntityManagerFactory emf = Persistence.createEntityManagerFactory(persistenceUnit);
static EntityManager entityManager = emf.createEntityManager();
// to run this app, uncomment the follow line in META-INF/persistence.xml
// <class>com.baeldung.optionalReturnType.UserOptionalField</class>
public static void main(String[] args) {
UserOptionalField user1 = new UserOptionalField();
user1.setUserId(1l);
user1.setFirstName(Optional.of("Bael Dung"));
entityManager.persist(user1);
UserOptional user2 = entityManager.find(UserOptional.class, 1l);
System.out.print("User2.firstName:" + user2.getFirstName());
}
}
@@ -0,0 +1,22 @@
package com.baeldung.optionalReturnType;
import javax.persistence.EntityManager;
import javax.persistence.EntityManagerFactory;
import javax.persistence.Persistence;
public class PersistOptionalTypeExample2 {
static String persistenceUnit = "com.baeldung.optionalReturnType";
static EntityManagerFactory emf = Persistence.createEntityManagerFactory(persistenceUnit);
static EntityManager em = emf.createEntityManager();
public static void main(String[] args) {
UserOptional user1 = new UserOptional();
user1.setUserId(1l);
user1.setFirstName("Bael Dung");
em.persist(user1);
UserOptional user2 = em.find(UserOptional.class, 1l);
System.out.print("User2.firstName:" + user2.getFirstName());
}
}
@@ -0,0 +1,22 @@
package com.baeldung.optionalReturnType;
import javax.persistence.EntityManager;
import javax.persistence.EntityManagerFactory;
import javax.persistence.Persistence;
public class PersistUserExample {
static String persistenceUnit = "com.baeldung.optionalReturnType";
static EntityManagerFactory emf = Persistence.createEntityManagerFactory(persistenceUnit);
static EntityManager em = emf.createEntityManager();
public static void main(String[] args) {
User user1 = new User();
user1.setUserId(1l);
user1.setFirstName("Bael Dung");
em.persist(user1);
User user2 = em.find(User.class, 1l);
System.out.print("User2.firstName:" + user2.getFirstName());
}
}
@@ -0,0 +1,41 @@
package com.baeldung.optionalReturnType;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.ObjectOutputStream;
import java.util.Optional;
public class SerializeOptionalTypeExample {
public static void main(String[] args) {
User user1 = new User();
user1.setUserId(1l);
user1.setFirstName("baeldung");
serializeObject(user1, "user1.ser");
UserOptionalField user2 = new UserOptionalField();
user2.setUserId(1l);
user2.setFirstName(Optional.of("baeldung"));
serializeObject(user2, "user2.ser");
}
public static void serializeObject(Object object, String fileName) {
// Serialization
try {
FileOutputStream file = new FileOutputStream(fileName);
ObjectOutputStream out = new ObjectOutputStream(file);
out.writeObject(object);
out.close();
file.close();
System.out.println("Object " + object.toString() + " has been serialized to file " + fileName);
} catch (IOException e) {
e.printStackTrace();
}
}
}
@@ -0,0 +1,31 @@
package com.baeldung.optionalReturnType;
import java.io.Serializable;
import javax.persistence.Entity;
import javax.persistence.Id;
@Entity
public class User implements Serializable {
@Id
private long userId;
private String firstName;
public long getUserId() {
return userId;
}
public void setUserId(long userId) {
this.userId = userId;
}
public String getFirstName() {
return firstName;
}
public void setFirstName(String firstName) {
this.firstName = firstName;
}
}
@@ -0,0 +1,35 @@
package com.baeldung.optionalReturnType;
import java.io.Serializable;
import java.util.Optional;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.Id;
@Entity
public class UserOptional implements Serializable {
@Id
private long userId;
@Column(nullable = true)
private String firstName;
public long getUserId() {
return userId;
}
public void setUserId(long userId) {
this.userId = userId;
}
public Optional<String> getFirstName() {
return Optional.ofNullable(firstName);
}
public void setFirstName(String firstName) {
this.firstName = firstName;
Optional.ofNullable(firstName);
}
}
@@ -0,0 +1,31 @@
package com.baeldung.optionalReturnType;
import java.io.Serializable;
import java.util.Optional;
import javax.persistence.Entity;
import javax.persistence.Id;
@Entity
public class UserOptionalField implements Serializable {
@Id
private long userId;
private Optional<String> firstName;
public long getUserId() {
return userId;
}
public void setUserId(long userId) {
this.userId = userId;
}
public Optional<String> getFirstName() {
return firstName;
}
public void setFirstName(Optional<String> firstName) {
this.firstName = firstName;
}
}
@@ -0,0 +1,34 @@
<?xml version="1.0" encoding="UTF-8"?>
<persistence xmlns="http://java.sun.com/xml/ns/persistence"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://java.sun.com/xml/ns/persistence http://java.sun.com/xml/ns/persistence/persistence_2_0.xsd"
version="2.0">
<persistence-unit
name="com.baeldung.optionalReturnType"
transaction-type="RESOURCE_LOCAL">
<description>Persist Optional Return Type Demo</description>
<provider>org.hibernate.jpa.HibernatePersistenceProvider</provider>
<class>com.baeldung.optionalReturnType.User</class>
<class>com.baeldung.optionalReturnType.UserOptional</class>
<!--
<class>com.baeldung.optionalReturnType.UserOptionalField</class>
-->
<exclude-unlisted-classes>true</exclude-unlisted-classes>
<properties>
<property name="javax.persistence.jdbc.driver"
value="org.h2.Driver" />
<property name="javax.persistence.jdbc.url"
value="jdbc:h2:mem:test" />
<property name="javax.persistence.jdbc.user" value="sa" />
<property name="javax.persistence.jdbc.password" value="" />
<property name="hibernate.dialect"
value="org.hibernate.dialect.H2Dialect" />
<property name="hibernate.hbm2ddl.auto" value="create-drop" />
<property name="show_sql" value="true" />
<property name="hibernate.temp.use_jdbc_metadata_defaults"
value="false" />
</properties>
</persistence-unit>
</persistence>
@@ -0,0 +1,26 @@
package com.baeldung.error;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
public class ErrorGeneratorUnitTest {
@Test(expected = AssertionError.class)
public void whenError_thenIsNotCaughtByCatchException() {
try {
throw new AssertionError();
} catch (Exception e) {
Assert.fail(); // errors are not caught by catch exception
}
}
@Test
public void whenError_thenIsCaughtByCatchError() {
try {
throw new AssertionError();
} catch (Error e) {
// caught! -> test pass
}
}
}
+61
View File
@@ -0,0 +1,61 @@
version: '3'
services:
## VOLUME CONTAINER-TO-CONTAINER AND HOST-TO-CONTAINER TEST ##
volumes-example-service:
image: alpine:latest
container_name: volumes-example-service
volumes:
- /tmp:/my-volumes/host-volume
- /home:/my-volumes/readonly-host-volume:ro
- my-named-global-volume:/my-volumes/named-global-volume
tty: true # Needed to keep the container running
another-volumes-example-service:
image: alpine:latest
container_name: another-volumes-example-service
volumes:
- my-named-global-volume:/another-path/the-same-named-global-volume
tty: true # Needed to keep the container running
## NETWORK CONTAINER-TO-CONTAINER TEST ##
network-example-service:
image: karthequian/helloworld:latest
container_name: network-example-service
networks:
- my-shared-network
another-service-in-the-same-network:
image: alpine:latest
container_name: another-service-in-the-same-network
networks:
- my-shared-network
tty: true # Needed to keep the container running
another-service-in-its-own-network:
image: alpine:latest
container_name: another-service-in-its-own-network
networks:
- my-private-network
tty: true # Needed to keep the container running
## NETWORK HOST-TO-CONTAINER TEST ##
network-example-service-available-to-host-on-port-1337:
image: karthequian/helloworld:latest
container_name: network-example-service-available-to-host-on-port-1337
networks:
- my-shared-network
ports:
- "1337:80"
volumes:
my-named-global-volume:
networks:
my-shared-network: {}
my-private-network: {}
@@ -15,8 +15,7 @@ public class ConvertToMap {
}
public Map<Integer, Book> listToMapWithDupKey(List<Book> books) {
return books.stream().collect(Collectors.toMap(Book::getReleaseYear, Function.identity(),
(o1, o2) -> o1));
return books.stream().collect(Collectors.toMap(Book::getReleaseYear, Function.identity(), (existing, replacement) -> existing));
}
public Map<Integer, Book> listToConcurrentMap(List<Book> books) {
@@ -2,6 +2,7 @@ package com.baeldung.convertToMap;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import static org.junit.Assert.*;
@@ -34,8 +35,10 @@ public class ConvertToMapUnitTest {
}
@Test
public void whenMapHasDuplicateKey_with_merge_function() {
assertTrue(convertToMap.listToMapWithDupKey(bookList).size() == 2);
public void whenMapHasDuplicateKeyThenMergeFunctionHandlesCollision() {
Map<Integer, Book> booksByYear = convertToMap.listToMapWithDupKey(bookList);
assertEquals(2, booksByYear.size());
assertEquals("0395489318", booksByYear.get(1954).getIsbn());
}
@Test
+7
View File
@@ -82,6 +82,12 @@
<scope>test</scope>
</dependency>
<dependency>
<groupId>edu.uci.ics</groupId>
<artifactId>crawler4j</artifactId>
<version>${crawler4j.version}</version>
</dependency
</dependencies>
<properties>
@@ -89,6 +95,7 @@
<classgraph.version>4.8.28</classgraph.version>
<jbpm.version>6.0.0.Final</jbpm.version>
<picocli.version>3.9.6</picocli.version>
<crawler4j.version>4.4.0</crawler4j.version>
<spring-boot-starter.version>2.1.4.RELEASE</spring-boot-starter.version>
</properties>
</project>
@@ -0,0 +1,22 @@
package com.baeldung.crawler4j;
public class CrawlerStatistics {
private int processedPageCount = 0;
private int totalLinksCount = 0;
public void incrementProcessedPageCount() {
processedPageCount++;
}
public void incrementTotalLinksCount(int linksCount) {
totalLinksCount += linksCount;
}
public int getProcessedPageCount() {
return processedPageCount;
}
public int getTotalLinksCount() {
return totalLinksCount;
}
}
@@ -0,0 +1,48 @@
package com.baeldung.crawler4j;
import java.util.Set;
import java.util.regex.Pattern;
import edu.uci.ics.crawler4j.crawler.Page;
import edu.uci.ics.crawler4j.crawler.WebCrawler;
import edu.uci.ics.crawler4j.parser.HtmlParseData;
import edu.uci.ics.crawler4j.url.WebURL;
public class HtmlCrawler extends WebCrawler {
private final static Pattern EXCLUSIONS = Pattern.compile(".*(\\.(css|js|xml|gif|jpg|png|mp3|mp4|zip|gz|pdf))$");
private CrawlerStatistics stats;
public HtmlCrawler(CrawlerStatistics stats) {
this.stats = stats;
}
@Override
public boolean shouldVisit(Page referringPage, WebURL url) {
String urlString = url.getURL().toLowerCase();
return !EXCLUSIONS.matcher(urlString).matches()
&& urlString.startsWith("https://www.baeldung.com/");
}
@Override
public void visit(Page page) {
String url = page.getWebURL().getURL();
stats.incrementProcessedPageCount();
if (page.getParseData() instanceof HtmlParseData) {
HtmlParseData htmlParseData = (HtmlParseData) page.getParseData();
String title = htmlParseData.getTitle();
String text = htmlParseData.getText();
String html = htmlParseData.getHtml();
Set<WebURL> links = htmlParseData.getOutgoingUrls();
stats.incrementTotalLinksCount(links.size());
System.out.printf("Page with title '%s' %n", title);
System.out.printf(" Text length: %d %n", text.length());
System.out.printf(" HTML length: %d %n", html.length());
System.out.printf(" %d outbound links %n", links.size());
}
}
}
@@ -0,0 +1,36 @@
package com.baeldung.crawler4j;
import java.io.File;
import edu.uci.ics.crawler4j.crawler.CrawlConfig;
import edu.uci.ics.crawler4j.crawler.CrawlController;
import edu.uci.ics.crawler4j.fetcher.PageFetcher;
import edu.uci.ics.crawler4j.robotstxt.RobotstxtConfig;
import edu.uci.ics.crawler4j.robotstxt.RobotstxtServer;
public class HtmlCrawlerController {
public static void main(String[] args) throws Exception {
File crawlStorage = new File("src/test/resources/crawler4j");
CrawlConfig config = new CrawlConfig();
config.setCrawlStorageFolder(crawlStorage.getAbsolutePath());
config.setMaxDepthOfCrawling(2);
int numCrawlers = 12;
PageFetcher pageFetcher = new PageFetcher(config);
RobotstxtConfig robotstxtConfig = new RobotstxtConfig();
RobotstxtServer robotstxtServer = new RobotstxtServer(robotstxtConfig, pageFetcher);
CrawlController controller = new CrawlController(config, pageFetcher, robotstxtServer);
controller.addSeed("https://www.baeldung.com/");
CrawlerStatistics stats = new CrawlerStatistics();
CrawlController.WebCrawlerFactory<HtmlCrawler> factory = () -> new HtmlCrawler(stats);
controller.start(factory, numCrawlers);
System.out.printf("Crawled %d pages %n", stats.getProcessedPageCount());
System.out.printf("Total Number of outbound links = %d %n", stats.getTotalLinksCount());
}
}
@@ -0,0 +1,49 @@
package com.baeldung.crawler4j;
import java.io.File;
import java.util.regex.Pattern;
import edu.uci.ics.crawler4j.crawler.Page;
import edu.uci.ics.crawler4j.crawler.WebCrawler;
import edu.uci.ics.crawler4j.parser.BinaryParseData;
import edu.uci.ics.crawler4j.url.WebURL;
public class ImageCrawler extends WebCrawler {
private final static Pattern EXCLUSIONS = Pattern.compile(".*(\\.(css|js|xml|gif|png|mp3|mp4|zip|gz|pdf))$");
private static final Pattern IMG_PATTERNS = Pattern.compile(".*(\\.(jpg|jpeg))$");
private File saveDir;
public ImageCrawler(File saveDir) {
this.saveDir = saveDir;
}
@Override
public boolean shouldVisit(Page referringPage, WebURL url) {
String urlString = url.getURL().toLowerCase();
if (EXCLUSIONS.matcher(urlString).matches()) {
return false;
}
if (IMG_PATTERNS.matcher(urlString).matches()
|| urlString.startsWith("https://www.baeldung.com/")) {
return true;
}
return false;
}
@Override
public void visit(Page page) {
String url = page.getWebURL().getURL();
if (IMG_PATTERNS.matcher(url).matches()
&& page.getParseData() instanceof BinaryParseData) {
String extension = url.substring(url.lastIndexOf("."));
int contentLength = page.getContentData().length;
System.out.printf("Extension is '%s' with content length %d %n", extension, contentLength);
}
}
}
@@ -0,0 +1,36 @@
package com.baeldung.crawler4j;
import java.io.File;
import edu.uci.ics.crawler4j.crawler.CrawlConfig;
import edu.uci.ics.crawler4j.crawler.CrawlController;
import edu.uci.ics.crawler4j.fetcher.PageFetcher;
import edu.uci.ics.crawler4j.robotstxt.RobotstxtConfig;
import edu.uci.ics.crawler4j.robotstxt.RobotstxtServer;
public class ImageCrawlerController {
public static void main(String[] args) throws Exception {
File crawlStorage = new File("src/test/resources/crawler4j");
CrawlConfig config = new CrawlConfig();
config.setCrawlStorageFolder(crawlStorage.getAbsolutePath());
config.setIncludeBinaryContentInCrawling(true);
config.setMaxPagesToFetch(500);
File saveDir = new File("src/test/resources/crawler4j");
int numCrawlers = 12;
PageFetcher pageFetcher = new PageFetcher(config);
RobotstxtConfig robotstxtConfig = new RobotstxtConfig();
RobotstxtServer robotstxtServer = new RobotstxtServer(robotstxtConfig, pageFetcher);
CrawlController controller = new CrawlController(config, pageFetcher, robotstxtServer);
controller.addSeed("https://www.baeldung.com/");
CrawlController.WebCrawlerFactory<ImageCrawler> factory = () -> new ImageCrawler(saveDir);
controller.start(factory, numCrawlers);
}
}
@@ -0,0 +1,54 @@
package com.baeldung.crawler4j;
import java.io.File;
import edu.uci.ics.crawler4j.crawler.CrawlConfig;
import edu.uci.ics.crawler4j.crawler.CrawlController;
import edu.uci.ics.crawler4j.fetcher.PageFetcher;
import edu.uci.ics.crawler4j.robotstxt.RobotstxtConfig;
import edu.uci.ics.crawler4j.robotstxt.RobotstxtServer;
public class MultipleCrawlerController {
public static void main(String[] args) throws Exception {
File crawlStorageBase = new File("src/test/resources/crawler4j");
CrawlConfig htmlConfig = new CrawlConfig();
CrawlConfig imageConfig = new CrawlConfig();
htmlConfig.setCrawlStorageFolder(new File(crawlStorageBase, "html").getAbsolutePath());
imageConfig.setCrawlStorageFolder(new File(crawlStorageBase, "image").getAbsolutePath());
imageConfig.setIncludeBinaryContentInCrawling(true);
htmlConfig.setMaxPagesToFetch(500);
imageConfig.setMaxPagesToFetch(1000);
PageFetcher pageFetcherHtml = new PageFetcher(htmlConfig);
PageFetcher pageFetcherImage = new PageFetcher(imageConfig);
RobotstxtConfig robotstxtConfig = new RobotstxtConfig();
RobotstxtServer robotstxtServer = new RobotstxtServer(robotstxtConfig, pageFetcherHtml);
CrawlController htmlController = new CrawlController(htmlConfig, pageFetcherHtml, robotstxtServer);
CrawlController imageController = new CrawlController(imageConfig, pageFetcherImage, robotstxtServer);
htmlController.addSeed("https://www.baeldung.com/");
imageController.addSeed("https://www.baeldung.com/");
CrawlerStatistics stats = new CrawlerStatistics();
CrawlController.WebCrawlerFactory<HtmlCrawler> htmlFactory = () -> new HtmlCrawler(stats);
File saveDir = new File("src/test/resources/crawler4j");
CrawlController.WebCrawlerFactory<ImageCrawler> imageFactory = () -> new ImageCrawler(saveDir);
imageController.startNonBlocking(imageFactory, 7);
htmlController.startNonBlocking(htmlFactory, 10);
htmlController.waitUntilFinish();
System.out.printf("Crawled %d pages %n", stats.getProcessedPageCount());
System.out.printf("Total Number of outbound links = %d %n", stats.getTotalLinksCount());
imageController.waitUntilFinish();
System.out.printf("Image Crawler is finished.");
}
}
+18 -13
View File
@@ -5,9 +5,9 @@
<parent>
<groupId>com.baeldung</groupId>
<artifactId>parent-modules</artifactId>
<artifactId>persistence-modules</artifactId>
<version>1.0.0-SNAPSHOT</version>
<relativePath>../../</relativePath>
<relativePath>..</relativePath>
</parent>
<artifactId>hibernate-mapping</artifactId>
@@ -31,22 +31,28 @@
<artifactId>h2</artifactId>
<version>${h2.version}</version>
</dependency>
<!-- validation -->
<!-- validation -->
<dependency>
<groupId>org.hibernate</groupId>
<artifactId>hibernate-validator</artifactId>
<version>${hibernate-validator.version}</version>
</dependency>
<dependency>
<groupId>javax.el</groupId>
<artifactId>javax.el-api</artifactId>
<version>${javax.el-api.version}</version>
</dependency>
<dependency>
<groupId>org.glassfish</groupId>
<artifactId>javax.el</artifactId>
<version>${org.glassfish.javax.el.version}</version>
</dependency>
<dependency>
<groupId>javax.money</groupId>
<artifactId>money-api</artifactId>
<version>1.0.3</version>
</dependency>
<dependency>
<groupId>org.javamoney</groupId>
<artifactId>moneta</artifactId>
<version>1.3</version>
<type>pom</type>
</dependency>
</dependencies>
<build>
@@ -60,11 +66,10 @@
</build>
<properties>
<hibernate.version>5.3.7.Final</hibernate.version>
<hibernate.version>5.3.10.Final</hibernate.version>
<assertj-core.version>3.8.0</assertj-core.version>
<hibernate-validator.version>5.3.3.Final</hibernate-validator.version>
<javax.el-api.version>2.2.5</javax.el-api.version>
<org.glassfish.javax.el.version>3.0.1-b08</org.glassfish.javax.el.version>
<hibernate-validator.version>6.0.16.Final</hibernate-validator.version>
<org.glassfish.javax.el.version>3.0.1-b11</org.glassfish.javax.el.version>
</properties>
</project>
</project>

Some files were not shown because too many files have changed in this diff Show More