Voomra/KinoSearch
0
Fork 0
Code Activity

Использование MongoDB

Browse Source
This commit is contained in:
DmitriyMX
2017-04-02 16:14:28 +03:00
parent b4a94e3dad
commit 8aaa2aa8e1
6 changed files with 93 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
pom.xml
View File

@@ -21,6 +21,7 @@
<java.version>1.8</java.version>
<slf4j.version>1.7.21</slf4j.version>
<spring.version>4.2.5.RELEASE</spring.version>
<spring.mongodb.version>1.10.1.RELEASE</spring.mongodb.version>
</properties>
<dependencies>
@@ -59,6 +60,11 @@
<artifactId>spring-context</artifactId>
<version>${spring.version}</version>
</dependency>
<dependency>
<groupId>org.springframework.data</groupId>
<artifactId>spring-data-mongodb</artifactId>
<version>${spring.mongodb.version}</version>
</dependency>
<!-- COMPONENTS -->
<dependency>

12
src/main/java/kinosearch/kinosearch3/cinema/onlinelife/ScannerImpl.java
View File

@@ -5,6 +5,7 @@
package kinosearch.kinosearch3.cinema.onlinelife;
import kinosearch.kinosearch3.browser.Browser;
import kinosearch.kinosearch3.spider.CinemaDocument;
import kinosearch.kinosearch3.spider.FileDownloader;
import kinosearch.kinosearch3.spider.ScannerCinema;
import org.jsoup.Jsoup;
@@ -13,6 +14,7 @@ import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.data.mongodb.core.MongoTemplate;
import java.io.File;
@@ -21,14 +23,16 @@ public class ScannerImpl implements ScannerCinema {
private FileDownloader fileDownloader;
private Browser browser;
private File saveToDir;
private MongoTemplate mongoTemplate;
public ScannerImpl(FileDownloader fileDownloader, Browser browser, File saveToDir) {
public ScannerImpl(FileDownloader fileDownloader, Browser browser, File saveToDir, MongoTemplate mongoTemplate) {
this.fileDownloader = fileDownloader;
this.browser = browser;
this.saveToDir = saveToDir;
if (!this.saveToDir.mkdirs() && !this.saveToDir.exists()) {
throw new IllegalStateException(String.format("dir not found: '%s'", this.saveToDir.getAbsolutePath()));
}
this.mongoTemplate = mongoTemplate;
}
@Override
@@ -54,7 +58,11 @@ public class ScannerImpl implements ScannerCinema {
String url = element.children().get(0).children().get(0).attr("src");
this.fileDownloader.addFile(url, new File(saveToDir, str + ".jpg"));
File saveToFile = new File(saveToDir, str + ".jpg");
this.fileDownloader.addFile(url, saveToFile);
CinemaDocument cinemaDocument = new CinemaDocument(saveToFile.getAbsolutePath(), this.getName());
this.mongoTemplate.save(cinemaDocument);
});
}
}

12
src/main/java/kinosearch/kinosearch3/cinema/seasonvar/ScannerImpl.java
View File

@@ -5,6 +5,7 @@
package kinosearch.kinosearch3.cinema.seasonvar;
import kinosearch.kinosearch3.browser.Browser;
import kinosearch.kinosearch3.spider.CinemaDocument;
import kinosearch.kinosearch3.spider.FileDownloader;
import kinosearch.kinosearch3.spider.ScannerCinema;
import org.jsoup.Jsoup;
@@ -12,6 +13,7 @@ import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.data.mongodb.core.MongoTemplate;
import java.io.File;
@@ -21,14 +23,16 @@ public class ScannerImpl implements ScannerCinema {
private final Browser browser;
private final File saveTo;
private int i = 1;
private MongoTemplate mongoTemplate;
public ScannerImpl(FileDownloader fileDownloader, Browser browser, File saveTo) {
public ScannerImpl(FileDownloader fileDownloader, Browser browser, File saveTo, MongoTemplate mongoTemplate) {
this.fileDownloader = fileDownloader;
this.browser = browser;
this.saveTo = saveTo;
if (!this.saveTo.mkdirs() && !this.saveTo.exists()) {
throw new IllegalStateException(String.format("dir not found: '%s'", this.saveTo.getAbsolutePath()));
}
this.mongoTemplate = mongoTemplate;
}
@Override
@@ -55,7 +59,11 @@ public class ScannerImpl implements ScannerCinema {
Document doc = Jsoup.parse(str2);
str2 = doc.getElementsByTag("img").get(0).attr("src");
this.fileDownloader.addFile(str2, new File(this.saveTo, str+".jpg"));
File saveToFile = new File(this.saveTo, str+".jpg");
this.fileDownloader.addFile(str2, saveToFile);
CinemaDocument cinemaDocument = new CinemaDocument(saveToFile.getAbsolutePath(), this.getName());
this.mongoTemplate.save(cinemaDocument);
});
}
}

42
src/main/java/kinosearch/kinosearch3/spider/CinemaDocument.java Normal file
View File

@@ -0,0 +1,42 @@
/*
* DmitriyMX <dimon550@gmail.com>
* 2017-04-02
*/
package kinosearch.kinosearch3.spider;
import org.springframework.data.annotation.Id;
import org.springframework.data.mongodb.core.mapping.Document;
@Document(collection = "cinema")
public class CinemaDocument {
@Id
private String id;
private String fileName;
private String typeWarez;
public CinemaDocument() {
}
public CinemaDocument(String fileName, String typeWarez) {
this.fileName = fileName;
this.typeWarez = typeWarez;
}
public String getFileName() {
return fileName;
}
public void setFileName(String fileName) {
this.fileName = fileName;
}
public String getTypeWarez() {
return typeWarez;
}
public void setTypeWarez(String typeWarez) {
this.typeWarez = typeWarez;
}
}

3
src/main/resources/application.properties Normal file
View File

@@ -0,0 +1,3 @@
mongo.host=127.0.0.1
mongo.port=27017
mongo.db=kinosearch

23
src/main/resources/kinosearch/kinosearch3/spider/spring.xml
View File

@@ -1,8 +1,27 @@
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:mongo="http://www.springframework.org/schema/data/mongo"
xsi:schemaLocation="http://www.springframework.org/schema/beans
http://www.springframework.org/schema/beans/spring-beans-3.0.xsd">
http://www.springframework.org/schema/beans/spring-beans-3.0.xsd
http://www.springframework.org/schema/data/mongo
http://www.springframework.org/schema/data/mongo/spring-mongo.xsd">
<bean id="configer" class="org.springframework.beans.factory.config.PropertyPlaceholderConfigurer">
<property name="fileEncoding" value="utf-8"/>
<property name="locations">
<list>
<value>classpath:application.properties</value>
</list>
</property>
</bean>
<mongo:mongo id="mongo" host="${mongo.host}" port="${mongo.port}"/>
<mongo:db-factory id="mongoDbFactory" dbname="${mongo.db}" mongo-ref="mongo"/>
<bean id="mongoTemplate" class="org.springframework.data.mongodb.core.MongoTemplate">
<constructor-arg name="mongoDbFactory" ref="mongoDbFactory"/>
</bean>
<bean id="fileDownloader" class="kinosearch.kinosearch3.spider.FileDownloader">
<constructor-arg name="threadCount" value="3"/>
@@ -21,12 +40,14 @@
</bean>
</constructor-arg>
<constructor-arg name="saveToDir" value="file:R:/onlinelife"/>
<constructor-arg name="mongoTemplate" ref="mongoTemplate"/>
</bean>
<bean id="seasonvarScanner" class="kinosearch.kinosearch3.cinema.seasonvar.ScannerImpl">
<constructor-arg name="fileDownloader" ref="fileDownloader"/>
<constructor-arg name="browser" ref="browser"/>
<constructor-arg name="saveTo" value="file:R:/seasonvar"/>
<constructor-arg name="mongoTemplate" ref="mongoTemplate"/>
</bean>
<bean id="spider" class="kinosearch.kinosearch3.spider.Spider">