Использование MongoDB

pom.xml

@@ -21,6 +21,7 @@
         <java.version>1.8</java.version>
         <slf4j.version>1.7.21</slf4j.version>
         <spring.version>4.2.5.RELEASE</spring.version>
+        <spring.mongodb.version>1.10.1.RELEASE</spring.mongodb.version>
     </properties>
 
     <dependencies>
@@ -59,6 +60,11 @@
             <artifactId>spring-context</artifactId>
             <version>${spring.version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.springframework.data</groupId>
+            <artifactId>spring-data-mongodb</artifactId>
+            <version>${spring.mongodb.version}</version>
+        </dependency>
 
         <!-- COMPONENTS -->
         <dependency>

src/main/java/kinosearch/kinosearch3/cinema/onlinelife/ScannerImpl.java

@@ -5,6 +5,7 @@
 package kinosearch.kinosearch3.cinema.onlinelife;
 
 import kinosearch.kinosearch3.browser.Browser;
+import kinosearch.kinosearch3.spider.CinemaDocument;
 import kinosearch.kinosearch3.spider.FileDownloader;
 import kinosearch.kinosearch3.spider.ScannerCinema;
 import org.jsoup.Jsoup;
@@ -13,6 +14,7 @@ import org.jsoup.nodes.Element;
 import org.jsoup.select.Elements;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.data.mongodb.core.MongoTemplate;
 
 import java.io.File;
 
@@ -21,14 +23,16 @@ public class ScannerImpl implements ScannerCinema {
     private FileDownloader fileDownloader;
     private Browser browser;
     private File saveToDir;
+    private MongoTemplate mongoTemplate;
 
-    public ScannerImpl(FileDownloader fileDownloader, Browser browser, File saveToDir) {
+    public ScannerImpl(FileDownloader fileDownloader, Browser browser, File saveToDir, MongoTemplate mongoTemplate) {
         this.fileDownloader = fileDownloader;
         this.browser = browser;
         this.saveToDir = saveToDir;
         if (!this.saveToDir.mkdirs() && !this.saveToDir.exists()) {
             throw new IllegalStateException(String.format("dir not found: '%s'", this.saveToDir.getAbsolutePath()));
         }
+        this.mongoTemplate = mongoTemplate;
     }
 
     @Override
@@ -54,7 +58,11 @@ public class ScannerImpl implements ScannerCinema {
 
                         String url = element.children().get(0).children().get(0).attr("src");
 
-                        this.fileDownloader.addFile(url, new File(saveToDir, str + ".jpg"));
+                        File saveToFile = new File(saveToDir, str + ".jpg");
+                        this.fileDownloader.addFile(url, saveToFile);
+
+                        CinemaDocument cinemaDocument = new CinemaDocument(saveToFile.getAbsolutePath(), this.getName());
+                        this.mongoTemplate.save(cinemaDocument);
                     });
         }
     }

src/main/java/kinosearch/kinosearch3/cinema/seasonvar/ScannerImpl.java

@@ -5,6 +5,7 @@
 package kinosearch.kinosearch3.cinema.seasonvar;
 
 import kinosearch.kinosearch3.browser.Browser;
+import kinosearch.kinosearch3.spider.CinemaDocument;
 import kinosearch.kinosearch3.spider.FileDownloader;
 import kinosearch.kinosearch3.spider.ScannerCinema;
 import org.jsoup.Jsoup;
@@ -12,6 +13,7 @@ import org.jsoup.nodes.Document;
 import org.jsoup.select.Elements;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.data.mongodb.core.MongoTemplate;
 
 import java.io.File;
 
@@ -21,14 +23,16 @@ public class ScannerImpl implements ScannerCinema {
     private final Browser browser;
     private final File saveTo;
     private int i = 1;
+    private MongoTemplate mongoTemplate;
 
-    public ScannerImpl(FileDownloader fileDownloader, Browser browser, File saveTo) {
+    public ScannerImpl(FileDownloader fileDownloader, Browser browser, File saveTo, MongoTemplate mongoTemplate) {
         this.fileDownloader = fileDownloader;
         this.browser = browser;
         this.saveTo = saveTo;
         if (!this.saveTo.mkdirs() && !this.saveTo.exists()) {
             throw new IllegalStateException(String.format("dir not found: '%s'", this.saveTo.getAbsolutePath()));
         }
+        this.mongoTemplate = mongoTemplate;
     }
 
     @Override
@@ -55,7 +59,11 @@ public class ScannerImpl implements ScannerCinema {
                     Document doc = Jsoup.parse(str2);
                     str2 = doc.getElementsByTag("img").get(0).attr("src");
 
-                    this.fileDownloader.addFile(str2, new File(this.saveTo, str+".jpg"));
+                    File saveToFile = new File(this.saveTo, str+".jpg");
+                    this.fileDownloader.addFile(str2, saveToFile);
+
+                    CinemaDocument cinemaDocument = new CinemaDocument(saveToFile.getAbsolutePath(), this.getName());
+                    this.mongoTemplate.save(cinemaDocument);
                 });
     }
 }

src/main/java/kinosearch/kinosearch3/spider/CinemaDocument.java

@@ -0,0 +1,42 @@
+/*
+ * DmitriyMX <dimon550@gmail.com>
+ * 2017-04-02
+ */
+package kinosearch.kinosearch3.spider;
+
+import org.springframework.data.annotation.Id;
+import org.springframework.data.mongodb.core.mapping.Document;
+
+@Document(collection = "cinema")
+public class CinemaDocument {
+    @Id
+    private String id;
+
+    private String fileName;
+
+    private String typeWarez;
+
+    public CinemaDocument() {
+    }
+
+    public CinemaDocument(String fileName, String typeWarez) {
+        this.fileName = fileName;
+        this.typeWarez = typeWarez;
+    }
+
+    public String getFileName() {
+        return fileName;
+    }
+
+    public void setFileName(String fileName) {
+        this.fileName = fileName;
+    }
+
+    public String getTypeWarez() {
+        return typeWarez;
+    }
+
+    public void setTypeWarez(String typeWarez) {
+        this.typeWarez = typeWarez;
+    }
+}

src/main/resources/application.properties

@@ -0,0 +1,3 @@
+mongo.host=127.0.0.1
+mongo.port=27017
+mongo.db=kinosearch

src/main/resources/kinosearch/kinosearch3/spider/spring.xml

@@ -1,8 +1,27 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <beans xmlns="http://www.springframework.org/schema/beans"
        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+       xmlns:mongo="http://www.springframework.org/schema/data/mongo"
        xsi:schemaLocation="http://www.springframework.org/schema/beans
-                           http://www.springframework.org/schema/beans/spring-beans-3.0.xsd">
+                           http://www.springframework.org/schema/beans/spring-beans-3.0.xsd
+                           http://www.springframework.org/schema/data/mongo
+                           http://www.springframework.org/schema/data/mongo/spring-mongo.xsd">
+
+    <bean id="configer" class="org.springframework.beans.factory.config.PropertyPlaceholderConfigurer">
+        <property name="fileEncoding" value="utf-8"/>
+        <property name="locations">
+            <list>
+                <value>classpath:application.properties</value>
+            </list>
+        </property>
+    </bean>
+
+    <mongo:mongo id="mongo" host="${mongo.host}" port="${mongo.port}"/>
+    <mongo:db-factory id="mongoDbFactory" dbname="${mongo.db}" mongo-ref="mongo"/>
+
+    <bean id="mongoTemplate" class="org.springframework.data.mongodb.core.MongoTemplate">
+        <constructor-arg name="mongoDbFactory" ref="mongoDbFactory"/>
+    </bean>
 
     <bean id="fileDownloader" class="kinosearch.kinosearch3.spider.FileDownloader">
         <constructor-arg name="threadCount" value="3"/>
@@ -21,12 +40,14 @@
             </bean>
         </constructor-arg>
         <constructor-arg name="saveToDir" value="file:R:/onlinelife"/>
+        <constructor-arg name="mongoTemplate" ref="mongoTemplate"/>
     </bean>
 
     <bean id="seasonvarScanner" class="kinosearch.kinosearch3.cinema.seasonvar.ScannerImpl">
         <constructor-arg name="fileDownloader" ref="fileDownloader"/>
         <constructor-arg name="browser" ref="browser"/>
         <constructor-arg name="saveTo" value="file:R:/seasonvar"/>
+        <constructor-arg name="mongoTemplate" ref="mongoTemplate"/>
     </bean>
 
     <bean id="spider" class="kinosearch.kinosearch3.spider.Spider">