diff --git a/libraries-2/pom.xml b/libraries-2/pom.xml index 6303c0cab5..218bb437cb 100644 --- a/libraries-2/pom.xml +++ b/libraries-2/pom.xml @@ -55,6 +55,11 @@ spring-boot-starter ${spring-boot-starter.version} + + edu.uci.ics + crawler4j + ${crawler4j.version} + @@ -62,6 +67,7 @@ 4.8.28 6.0.0.Final 3.9.6 + 4.4.0 2.1.4.RELEASE diff --git a/libraries-2/src/main/java/com/baeldung/crawler4j/CrawlerStatistics.java b/libraries-2/src/main/java/com/baeldung/crawler4j/CrawlerStatistics.java new file mode 100644 index 0000000000..e3237a2755 --- /dev/null +++ b/libraries-2/src/main/java/com/baeldung/crawler4j/CrawlerStatistics.java @@ -0,0 +1,22 @@ +package com.baeldung.crawler4j; + +public class CrawlerStatistics { + private int processedPageCount = 0; + private int totalLinksCount = 0; + + public void incrementProcessedPageCount() { + processedPageCount++; + } + + public void incrementTotalLinksCount(int linksCount) { + totalLinksCount += linksCount; + } + + public int getProcessedPageCount() { + return processedPageCount; + } + + public int getTotalLinksCount() { + return totalLinksCount; + } +} diff --git a/libraries-2/src/main/java/com/baeldung/crawler4j/HtmlCrawler.java b/libraries-2/src/main/java/com/baeldung/crawler4j/HtmlCrawler.java new file mode 100644 index 0000000000..b77e1e075f --- /dev/null +++ b/libraries-2/src/main/java/com/baeldung/crawler4j/HtmlCrawler.java @@ -0,0 +1,48 @@ +package com.baeldung.crawler4j; + +import java.util.Set; +import java.util.regex.Pattern; + +import edu.uci.ics.crawler4j.crawler.Page; +import edu.uci.ics.crawler4j.crawler.WebCrawler; +import edu.uci.ics.crawler4j.parser.HtmlParseData; +import edu.uci.ics.crawler4j.url.WebURL; + +public class HtmlCrawler extends WebCrawler { + + private final static Pattern EXCLUSIONS = Pattern.compile(".*(\\.(css|js|xml|gif|jpg|png|mp3|mp4|zip|gz|pdf))$"); + + private CrawlerStatistics stats; + + public HtmlCrawler(CrawlerStatistics stats) { + this.stats = stats; + } + + @Override + public boolean shouldVisit(Page referringPage, WebURL url) { + String urlString = url.getURL().toLowerCase(); + return !EXCLUSIONS.matcher(urlString).matches() + && urlString.startsWith("https://www.baeldung.com/"); + } + + @Override + public void visit(Page page) { + String url = page.getWebURL().getURL(); + stats.incrementProcessedPageCount(); + + if (page.getParseData() instanceof HtmlParseData) { + HtmlParseData htmlParseData = (HtmlParseData) page.getParseData(); + String title = htmlParseData.getTitle(); + String text = htmlParseData.getText(); + String html = htmlParseData.getHtml(); + Set links = htmlParseData.getOutgoingUrls(); + stats.incrementTotalLinksCount(links.size()); + + System.out.printf("Page with title '%s' %n", title); + System.out.printf(" Text length: %d %n", text.length()); + System.out.printf(" HTML length: %d %n", html.length()); + System.out.printf(" %d outbound links %n", links.size()); + } + } + +} diff --git a/libraries-2/src/main/java/com/baeldung/crawler4j/HtmlCrawlerController.java b/libraries-2/src/main/java/com/baeldung/crawler4j/HtmlCrawlerController.java new file mode 100644 index 0000000000..82c1c6bdd7 --- /dev/null +++ b/libraries-2/src/main/java/com/baeldung/crawler4j/HtmlCrawlerController.java @@ -0,0 +1,36 @@ +package com.baeldung.crawler4j; + +import java.io.File; + +import edu.uci.ics.crawler4j.crawler.CrawlConfig; +import edu.uci.ics.crawler4j.crawler.CrawlController; +import edu.uci.ics.crawler4j.fetcher.PageFetcher; +import edu.uci.ics.crawler4j.robotstxt.RobotstxtConfig; +import edu.uci.ics.crawler4j.robotstxt.RobotstxtServer; + +public class HtmlCrawlerController { + + public static void main(String[] args) throws Exception { + File crawlStorage = new File("src/test/resources/crawler4j"); + CrawlConfig config = new CrawlConfig(); + config.setCrawlStorageFolder(crawlStorage.getAbsolutePath()); + config.setMaxDepthOfCrawling(2); + + int numCrawlers = 12; + + PageFetcher pageFetcher = new PageFetcher(config); + RobotstxtConfig robotstxtConfig = new RobotstxtConfig(); + RobotstxtServer robotstxtServer = new RobotstxtServer(robotstxtConfig, pageFetcher); + CrawlController controller = new CrawlController(config, pageFetcher, robotstxtServer); + + controller.addSeed("https://www.baeldung.com/"); + + CrawlerStatistics stats = new CrawlerStatistics(); + CrawlController.WebCrawlerFactory factory = () -> new HtmlCrawler(stats); + + controller.start(factory, numCrawlers); + System.out.printf("Crawled %d pages %n", stats.getProcessedPageCount()); + System.out.printf("Total Number of outbound links = %d %n", stats.getTotalLinksCount()); + } + +} diff --git a/libraries-2/src/main/java/com/baeldung/crawler4j/ImageCrawler.java b/libraries-2/src/main/java/com/baeldung/crawler4j/ImageCrawler.java new file mode 100644 index 0000000000..ebb5d96f36 --- /dev/null +++ b/libraries-2/src/main/java/com/baeldung/crawler4j/ImageCrawler.java @@ -0,0 +1,49 @@ +package com.baeldung.crawler4j; + +import java.io.File; +import java.util.regex.Pattern; + +import edu.uci.ics.crawler4j.crawler.Page; +import edu.uci.ics.crawler4j.crawler.WebCrawler; +import edu.uci.ics.crawler4j.parser.BinaryParseData; +import edu.uci.ics.crawler4j.url.WebURL; + +public class ImageCrawler extends WebCrawler { + private final static Pattern EXCLUSIONS = Pattern.compile(".*(\\.(css|js|xml|gif|png|mp3|mp4|zip|gz|pdf))$"); + + private static final Pattern IMG_PATTERNS = Pattern.compile(".*(\\.(jpg|jpeg))$"); + + private File saveDir; + + public ImageCrawler(File saveDir) { + this.saveDir = saveDir; + } + + @Override + public boolean shouldVisit(Page referringPage, WebURL url) { + String urlString = url.getURL().toLowerCase(); + if (EXCLUSIONS.matcher(urlString).matches()) { + return false; + } + + if (IMG_PATTERNS.matcher(urlString).matches() + || urlString.startsWith("https://www.baeldung.com/")) { + return true; + } + + return false; + } + + @Override + public void visit(Page page) { + String url = page.getWebURL().getURL(); + if (IMG_PATTERNS.matcher(url).matches() + && page.getParseData() instanceof BinaryParseData) { + String extension = url.substring(url.lastIndexOf(".")); + int contentLength = page.getContentData().length; + + System.out.printf("Extension is '%s' with content length %d %n", extension, contentLength); + } + } + +} diff --git a/libraries-2/src/main/java/com/baeldung/crawler4j/ImageCrawlerController.java b/libraries-2/src/main/java/com/baeldung/crawler4j/ImageCrawlerController.java new file mode 100644 index 0000000000..9db32f1bfb --- /dev/null +++ b/libraries-2/src/main/java/com/baeldung/crawler4j/ImageCrawlerController.java @@ -0,0 +1,36 @@ +package com.baeldung.crawler4j; + +import java.io.File; + +import edu.uci.ics.crawler4j.crawler.CrawlConfig; +import edu.uci.ics.crawler4j.crawler.CrawlController; +import edu.uci.ics.crawler4j.fetcher.PageFetcher; +import edu.uci.ics.crawler4j.robotstxt.RobotstxtConfig; +import edu.uci.ics.crawler4j.robotstxt.RobotstxtServer; + +public class ImageCrawlerController { + + public static void main(String[] args) throws Exception { + File crawlStorage = new File("src/test/resources/crawler4j"); + CrawlConfig config = new CrawlConfig(); + config.setCrawlStorageFolder(crawlStorage.getAbsolutePath()); + config.setIncludeBinaryContentInCrawling(true); + config.setMaxPagesToFetch(500); + + File saveDir = new File("src/test/resources/crawler4j"); + + int numCrawlers = 12; + + PageFetcher pageFetcher = new PageFetcher(config); + RobotstxtConfig robotstxtConfig = new RobotstxtConfig(); + RobotstxtServer robotstxtServer = new RobotstxtServer(robotstxtConfig, pageFetcher); + CrawlController controller = new CrawlController(config, pageFetcher, robotstxtServer); + + controller.addSeed("https://www.baeldung.com/"); + + CrawlController.WebCrawlerFactory factory = () -> new ImageCrawler(saveDir); + + controller.start(factory, numCrawlers); + } + +} diff --git a/libraries-2/src/main/java/com/baeldung/crawler4j/MultipleCrawlerController.java b/libraries-2/src/main/java/com/baeldung/crawler4j/MultipleCrawlerController.java new file mode 100644 index 0000000000..7662607f80 --- /dev/null +++ b/libraries-2/src/main/java/com/baeldung/crawler4j/MultipleCrawlerController.java @@ -0,0 +1,54 @@ +package com.baeldung.crawler4j; + +import java.io.File; + +import edu.uci.ics.crawler4j.crawler.CrawlConfig; +import edu.uci.ics.crawler4j.crawler.CrawlController; +import edu.uci.ics.crawler4j.fetcher.PageFetcher; +import edu.uci.ics.crawler4j.robotstxt.RobotstxtConfig; +import edu.uci.ics.crawler4j.robotstxt.RobotstxtServer; + +public class MultipleCrawlerController { + public static void main(String[] args) throws Exception { + File crawlStorageBase = new File("src/test/resources/crawler4j"); + CrawlConfig htmlConfig = new CrawlConfig(); + CrawlConfig imageConfig = new CrawlConfig(); + + htmlConfig.setCrawlStorageFolder(new File(crawlStorageBase, "html").getAbsolutePath()); + imageConfig.setCrawlStorageFolder(new File(crawlStorageBase, "image").getAbsolutePath()); + imageConfig.setIncludeBinaryContentInCrawling(true); + + htmlConfig.setMaxPagesToFetch(500); + imageConfig.setMaxPagesToFetch(1000); + + PageFetcher pageFetcherHtml = new PageFetcher(htmlConfig); + PageFetcher pageFetcherImage = new PageFetcher(imageConfig); + + RobotstxtConfig robotstxtConfig = new RobotstxtConfig(); + RobotstxtServer robotstxtServer = new RobotstxtServer(robotstxtConfig, pageFetcherHtml); + + CrawlController htmlController = new CrawlController(htmlConfig, pageFetcherHtml, robotstxtServer); + CrawlController imageController = new CrawlController(imageConfig, pageFetcherImage, robotstxtServer); + + htmlController.addSeed("https://www.baeldung.com/"); + imageController.addSeed("https://www.baeldung.com/"); + + CrawlerStatistics stats = new CrawlerStatistics(); + CrawlController.WebCrawlerFactory htmlFactory = () -> new HtmlCrawler(stats); + + File saveDir = new File("src/test/resources/crawler4j"); + CrawlController.WebCrawlerFactory imageFactory = () -> new ImageCrawler(saveDir); + + imageController.startNonBlocking(imageFactory, 7); + htmlController.startNonBlocking(htmlFactory, 10); + + + htmlController.waitUntilFinish(); + System.out.printf("Crawled %d pages %n", stats.getProcessedPageCount()); + System.out.printf("Total Number of outbound links = %d %n", stats.getTotalLinksCount()); + + imageController.waitUntilFinish(); + System.out.printf("Image Crawler is finished."); + + } +}