diff --git a/pom.xml b/pom.xml index ec74edc..ffb7d02 100644 --- a/pom.xml +++ b/pom.xml @@ -21,6 +21,7 @@ 1.8 1.7.21 4.2.5.RELEASE + 1.10.1.RELEASE @@ -59,6 +60,11 @@ spring-context ${spring.version} + + org.springframework.data + spring-data-mongodb + ${spring.mongodb.version} + diff --git a/src/main/java/kinosearch/kinosearch3/cinema/onlinelife/ScannerImpl.java b/src/main/java/kinosearch/kinosearch3/cinema/onlinelife/ScannerImpl.java index 380d999..90576c6 100644 --- a/src/main/java/kinosearch/kinosearch3/cinema/onlinelife/ScannerImpl.java +++ b/src/main/java/kinosearch/kinosearch3/cinema/onlinelife/ScannerImpl.java @@ -5,6 +5,7 @@ package kinosearch.kinosearch3.cinema.onlinelife; import kinosearch.kinosearch3.browser.Browser; +import kinosearch.kinosearch3.spider.CinemaDocument; import kinosearch.kinosearch3.spider.FileDownloader; import kinosearch.kinosearch3.spider.ScannerCinema; import org.jsoup.Jsoup; @@ -13,6 +14,7 @@ import org.jsoup.nodes.Element; import org.jsoup.select.Elements; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.data.mongodb.core.MongoTemplate; import java.io.File; @@ -21,14 +23,16 @@ public class ScannerImpl implements ScannerCinema { private FileDownloader fileDownloader; private Browser browser; private File saveToDir; + private MongoTemplate mongoTemplate; - public ScannerImpl(FileDownloader fileDownloader, Browser browser, File saveToDir) { + public ScannerImpl(FileDownloader fileDownloader, Browser browser, File saveToDir, MongoTemplate mongoTemplate) { this.fileDownloader = fileDownloader; this.browser = browser; this.saveToDir = saveToDir; if (!this.saveToDir.mkdirs() && !this.saveToDir.exists()) { throw new IllegalStateException(String.format("dir not found: '%s'", this.saveToDir.getAbsolutePath())); } + this.mongoTemplate = mongoTemplate; } @Override @@ -54,7 +58,11 @@ public class ScannerImpl implements ScannerCinema { String url = element.children().get(0).children().get(0).attr("src"); - this.fileDownloader.addFile(url, new File(saveToDir, str + ".jpg")); + File saveToFile = new File(saveToDir, str + ".jpg"); + this.fileDownloader.addFile(url, saveToFile); + + CinemaDocument cinemaDocument = new CinemaDocument(saveToFile.getAbsolutePath(), this.getName()); + this.mongoTemplate.save(cinemaDocument); }); } } diff --git a/src/main/java/kinosearch/kinosearch3/cinema/seasonvar/ScannerImpl.java b/src/main/java/kinosearch/kinosearch3/cinema/seasonvar/ScannerImpl.java index fcfb951..fc4eccc 100644 --- a/src/main/java/kinosearch/kinosearch3/cinema/seasonvar/ScannerImpl.java +++ b/src/main/java/kinosearch/kinosearch3/cinema/seasonvar/ScannerImpl.java @@ -5,6 +5,7 @@ package kinosearch.kinosearch3.cinema.seasonvar; import kinosearch.kinosearch3.browser.Browser; +import kinosearch.kinosearch3.spider.CinemaDocument; import kinosearch.kinosearch3.spider.FileDownloader; import kinosearch.kinosearch3.spider.ScannerCinema; import org.jsoup.Jsoup; @@ -12,6 +13,7 @@ import org.jsoup.nodes.Document; import org.jsoup.select.Elements; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.data.mongodb.core.MongoTemplate; import java.io.File; @@ -21,14 +23,16 @@ public class ScannerImpl implements ScannerCinema { private final Browser browser; private final File saveTo; private int i = 1; + private MongoTemplate mongoTemplate; - public ScannerImpl(FileDownloader fileDownloader, Browser browser, File saveTo) { + public ScannerImpl(FileDownloader fileDownloader, Browser browser, File saveTo, MongoTemplate mongoTemplate) { this.fileDownloader = fileDownloader; this.browser = browser; this.saveTo = saveTo; if (!this.saveTo.mkdirs() && !this.saveTo.exists()) { throw new IllegalStateException(String.format("dir not found: '%s'", this.saveTo.getAbsolutePath())); } + this.mongoTemplate = mongoTemplate; } @Override @@ -55,7 +59,11 @@ public class ScannerImpl implements ScannerCinema { Document doc = Jsoup.parse(str2); str2 = doc.getElementsByTag("img").get(0).attr("src"); - this.fileDownloader.addFile(str2, new File(this.saveTo, str+".jpg")); + File saveToFile = new File(this.saveTo, str+".jpg"); + this.fileDownloader.addFile(str2, saveToFile); + + CinemaDocument cinemaDocument = new CinemaDocument(saveToFile.getAbsolutePath(), this.getName()); + this.mongoTemplate.save(cinemaDocument); }); } } diff --git a/src/main/java/kinosearch/kinosearch3/spider/CinemaDocument.java b/src/main/java/kinosearch/kinosearch3/spider/CinemaDocument.java new file mode 100644 index 0000000..88642a6 --- /dev/null +++ b/src/main/java/kinosearch/kinosearch3/spider/CinemaDocument.java @@ -0,0 +1,42 @@ +/* + * DmitriyMX + * 2017-04-02 + */ +package kinosearch.kinosearch3.spider; + +import org.springframework.data.annotation.Id; +import org.springframework.data.mongodb.core.mapping.Document; + +@Document(collection = "cinema") +public class CinemaDocument { + @Id + private String id; + + private String fileName; + + private String typeWarez; + + public CinemaDocument() { + } + + public CinemaDocument(String fileName, String typeWarez) { + this.fileName = fileName; + this.typeWarez = typeWarez; + } + + public String getFileName() { + return fileName; + } + + public void setFileName(String fileName) { + this.fileName = fileName; + } + + public String getTypeWarez() { + return typeWarez; + } + + public void setTypeWarez(String typeWarez) { + this.typeWarez = typeWarez; + } +} diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties new file mode 100644 index 0000000..96f8981 --- /dev/null +++ b/src/main/resources/application.properties @@ -0,0 +1,3 @@ +mongo.host=127.0.0.1 +mongo.port=27017 +mongo.db=kinosearch \ No newline at end of file diff --git a/src/main/resources/kinosearch/kinosearch3/spider/spring.xml b/src/main/resources/kinosearch/kinosearch3/spider/spring.xml index dd52712..f75f6f8 100644 --- a/src/main/resources/kinosearch/kinosearch3/spider/spring.xml +++ b/src/main/resources/kinosearch/kinosearch3/spider/spring.xml @@ -1,8 +1,27 @@ + http://www.springframework.org/schema/beans/spring-beans-3.0.xsd + http://www.springframework.org/schema/data/mongo + http://www.springframework.org/schema/data/mongo/spring-mongo.xsd"> + + + + + + classpath:application.properties + + + + + + + + + + @@ -21,12 +40,14 @@ + +