Voomra/KinoSearch
0
Fork 0
Code Activity

Перепроектируем классы для возможности тестирования

Browse Source
This commit is contained in:
DmitriyMX
2017-04-07 01:33:10 +03:00
parent 82f26515ef
commit 369ba8aa6c
14 changed files with 436 additions and 160 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
src/main/java/kinosearch/kinosearch3/cinema/onlinelife/ScannerImpl.java
View File

@@ -5,6 +5,7 @@
package kinosearch.kinosearch3.cinema.onlinelife;
import kinosearch.kinosearch3.browser.Browser;
import kinosearch.kinosearch3.spider.BaseRepository;
import kinosearch.kinosearch3.spider.CinemaDocument;
import kinosearch.kinosearch3.spider.FileDownloader;
import kinosearch.kinosearch3.spider.ScannerCinema;
@@ -12,27 +13,23 @@ import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.data.mongodb.core.MongoTemplate;
import java.io.File;
public class ScannerImpl implements ScannerCinema {
private Logger logger = LoggerFactory.getLogger(ScannerImpl.class);
private FileDownloader fileDownloader;
private Browser browser;
private File saveToDir;
private MongoTemplate mongoTemplate;
private BaseRepository repository;
public ScannerImpl(FileDownloader fileDownloader, Browser browser, File saveToDir, MongoTemplate mongoTemplate) {
public ScannerImpl(FileDownloader fileDownloader, Browser browser, File saveToDir, BaseRepository repository) {
this.fileDownloader = fileDownloader;
this.browser = browser;
this.saveToDir = saveToDir;
if (!this.saveToDir.mkdirs() && !this.saveToDir.exists()) {
throw new IllegalStateException(String.format("dir not found: '%s'", this.saveToDir.getAbsolutePath()));
}
this.mongoTemplate = mongoTemplate;
this.repository = repository;
}
@Override
@@ -45,30 +42,32 @@ public class ScannerImpl implements ScannerCinema {
int lastPage = getLastPage();
for (int page = lastPage; page > 0; page--) {
logger.info("page #{}...", page);
String html = browser.get(String.format("http://www.online-life.cc/filmy/zarubezh-filmy/page/%d/", page));
String html = browser.get(String.format("http://www.online-life.cc/lastnews/page/%d/", page));
Document document = Jsoup.parse(html);
Elements elements = document.getElementsByClass("slider-item");
elements.stream()
.map(element -> element.children().get(0))
.forEach(element -> {
String str = element.attr("href");
str = str.substring(str.lastIndexOf("/") + 1, str.lastIndexOf("."));
String url = element.children().get(0).children().get(0).attr("src");
File saveToFile = new File(saveToDir, str + ".jpg");
this.fileDownloader.addFile(url, saveToFile);
CinemaDocument cinemaDocument = new CinemaDocument(saveToFile.getAbsolutePath(), this.getName());
this.mongoTemplate.save(cinemaDocument);
});
.flatMap(element -> element.children().stream())
.filter(element -> element.tagName().equals("a"))
.map(element -> element.attr("href"))
.forEach(this::browseAndSave);
}
}
@Override
public void browseAndSave(String url) {
Document document = Jsoup.parse(browser.get(url));
Element element = document.getElementsByClass("full-poster").get(0);
File saveTo = new File(this.saveToDir, url.substring(url.lastIndexOf("/")+1, url.lastIndexOf("."))+".jpg");
this.fileDownloader.addFile(element.attr("src"), saveTo);
CinemaDocument cinemaDocument = new CinemaDocument(saveTo.getAbsolutePath(), this.getName());
repository.save(cinemaDocument);
}
private int getLastPage() {
String html = browser.get("http://www.online-life.cc/filmy/zarubezh-filmy/");
String html = browser.get("http://www.online-life.cc/lastnews/");
Document document = Jsoup.parse(html);
Elements elements = document.getElementsByClass("navigation");

45
src/main/java/kinosearch/kinosearch3/cinema/seasonvar/ScannerImpl.java
View File

@@ -5,34 +5,32 @@
package kinosearch.kinosearch3.cinema.seasonvar;
import kinosearch.kinosearch3.browser.Browser;
import kinosearch.kinosearch3.spider.BaseRepository;
import kinosearch.kinosearch3.spider.CinemaDocument;
import kinosearch.kinosearch3.spider.FileDownloader;
import kinosearch.kinosearch3.spider.ScannerCinema;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.data.mongodb.core.MongoTemplate;
import java.io.File;
public class ScannerImpl implements ScannerCinema {
private Logger logger = LoggerFactory.getLogger(ScannerImpl.class);
private static final String DOMAIN = "http://seasonvar.ru/";
private final FileDownloader fileDownloader;
private final Browser browser;
private final File saveTo;
private int i = 1;
private MongoTemplate mongoTemplate;
private final BaseRepository repository;
public ScannerImpl(FileDownloader fileDownloader, Browser browser, File saveTo, MongoTemplate mongoTemplate) {
public ScannerImpl(FileDownloader fileDownloader, Browser browser, File saveTo, BaseRepository repository) {
this.fileDownloader = fileDownloader;
this.browser = browser;
this.saveTo = saveTo;
if (!this.saveTo.mkdirs() && !this.saveTo.exists()) {
throw new IllegalStateException(String.format("dir not found: '%s'", this.saveTo.getAbsolutePath()));
}
this.mongoTemplate = mongoTemplate;
this.repository = repository;
}
@Override
@@ -42,28 +40,25 @@ public class ScannerImpl implements ScannerCinema {
@Override
public void run() {
String html = browser.get("http://seasonvar.ru/");
String html = browser.get(DOMAIN);
Document document = Jsoup.parse(html);
Elements elements = document.getElementsByClass("betterT");
Elements elements = document.getElementsByAttribute("data-tabgr");
elements.stream()
.filter(element -> element.tagName().equals("div"))
.map(element -> element.children().get(0))
.forEach(element -> {
logger.info("element #{}", i++);
String str = element.attr("href");
str = str.substring(1, str.lastIndexOf("."));
.flatMap(element -> element.children().stream())
.map(element -> element.attr("href"))
.forEach(url -> this.browseAndSave(DOMAIN + url));
}
String str2 = element.attr("data");
str2 = browser.get("http://seasonvar.ru"+str2);
Document doc = Jsoup.parse(str2);
str2 = doc.getElementsByTag("img").get(0).attr("src");
@Override
public void browseAndSave(String url) {
Document document = Jsoup.parse(browser.get(url));
Element element = document.getElementsByAttributeValue("itemprop", "thumbnailUrl").get(0);
File saveToFile = new File(this.saveTo, str+".jpg");
this.fileDownloader.addFile(str2, saveToFile);
File saveTo = new File(this.saveTo, url.substring(url.indexOf("/")+1, url.lastIndexOf("."))+".jpg");
this.fileDownloader.addFile(element.attr("src"), saveTo);
CinemaDocument cinemaDocument = new CinemaDocument(saveToFile.getAbsolutePath(), this.getName());
this.mongoTemplate.save(cinemaDocument);
});
CinemaDocument cinemaDocument = new CinemaDocument(saveTo.getAbsolutePath(), this.getName());
this.repository.save(cinemaDocument);
}
}