Voomra/KinoSearch
0
Fork 0
Code Activity

Подготовка Spider-а к релизу

Browse Source
This commit is contained in:
DmitriyMX
2017-04-07 18:11:50 +03:00
parent 8e4122afe0
commit f502665234
7 changed files with 51 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
src/main/java/kinosearch/kinosearch3/cinema/OnlinelifeScanner.java
View File

@@ -13,10 +13,13 @@ import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
public class OnlinelifeScanner implements ScannerCinema {
private final Logger logger = LoggerFactory.getLogger(OnlinelifeScanner.class);
private FileDownloader fileDownloader;
private Browser browser;
private File saveToDir;
@@ -47,7 +50,10 @@ public class OnlinelifeScanner implements ScannerCinema {
.flatMap(element -> element.children().stream())
.filter(element -> element.tagName().equals("a"))
.map(element -> element.attr("href"))
.forEach(this::browseAndSave);
.forEach(url -> {
logger.info(url);
this.browseAndSave(url);
});
}
}

12
src/main/java/kinosearch/kinosearch3/cinema/SeasonvarScanner.java
View File

@@ -13,11 +13,14 @@ import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
public class SeasonvarScanner implements ScannerCinema {
private static final String DOMAIN = "http://seasonvar.ru/";
private static final String DOMAIN = "http://seasonvar.ru";
private final Logger logger = LoggerFactory.getLogger(SeasonvarScanner.class);
private final FileDownloader fileDownloader;
private final Browser browser;
private final File saveTo;
@@ -44,7 +47,10 @@ public class SeasonvarScanner implements ScannerCinema {
elements.stream()
.flatMap(element -> element.children().stream())
.map(element -> element.attr("href"))
.forEach(url -> this.browseAndSave(DOMAIN + url));
.forEach(url -> {
logger.info(DOMAIN + url);
this.browseAndSave(DOMAIN + url);
});
}
@Override
@@ -52,7 +58,7 @@ public class SeasonvarScanner implements ScannerCinema {
Document document = Jsoup.parse(browser.get(url));
Element element = document.getElementsByAttributeValue("itemprop", "thumbnailUrl").get(0);
File saveTo = new File(this.saveTo, url.substring(url.indexOf("/")+1, url.lastIndexOf("."))+".jpg");
File saveTo = new File(this.saveTo, url.substring(url.lastIndexOf("/")+1, url.lastIndexOf("."))+".jpg");
this.fileDownloader.addFile(element.attr("src"), saveTo);
CinemaDocument cinemaDocument = new CinemaDocument();