Crawler4j Java多线程Web爬虫开源项目

Crawler4j是一个开源的Java类库提供一个用于抓取Web页面的简单接口。可以利用它来构建一个多线程的Web爬虫。

示例代码：

importjava.util.ArrayList;importjava.util.regex.Pattern;importedu.uci.ics.crawler4j.crawler.Page;importedu.uci.ics.crawler4j.crawler.WebCrawler;importedu.uci.ics.crawler4j.url.WebURL;publicclassMyCrawlerextendsWebCrawler{       Patternfilters=Pattern.compile(".*(\\.(css|js|bmp|gif|jpe?g"               +"|png|tiff?|mid|mp2|mp3|mp4"               +"|wav|avi|mov|mpeg|ram|m4v|pdf"               +"|rm|smil|wmv|swf|wma|zip|rar|gz))$");              publicMyCrawler(){       }       publicbooleanshouldVisit(WebURLurl){               Stringhref=url.getURL().toLowerCase();               if(filters.matcher(href).matches()){                       returnfalse;               }               if(href.startsWith("https://www.ics.uci.edu/")){                       returntrue;               }               returnfalse;       }       publicvoidvisit(Pagepage){               intdocid=page.getWebURL().getDocid();               Stringurl=page.getWebURL().getURL();                       Stringtext=page.getText();               ArrayList<WebURL>links=page.getURLs();                     }}