From 8e4122afe0a5e1892985ac43051d6a3b0b1a91d1 Mon Sep 17 00:00:00 2001 From: DmitriyMX Date: Fri, 7 Apr 2017 15:30:37 +0300 Subject: [PATCH] =?UTF-8?q?=D0=92=20=D0=B1=D0=B0=D0=B7=D1=83=20=D0=B7?= =?UTF-8?q?=D0=B0=D0=BD=D0=BE=D1=81=D0=B8=D1=82=D1=81=D1=8F=20=D0=BD=D0=B0?= =?UTF-8?q?=D0=B7=D0=B2=D0=B0=D0=BD=D0=B8=D0=B5=20=D0=B8=20=D0=BE=D0=BF?= =?UTF-8?q?=D0=B8=D1=81=D0=B0=D0=BD=D0=B8=D0=B5=20=D0=BA=D0=B8=D0=BD=D0=BE?= =?UTF-8?q?/=D1=81=D0=B5=D1=80=D0=B8=D0=B0=D0=BB=D0=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../kinosearch3/cinema/OnlinelifeScanner.java | 6 +++++- .../kinosearch3/cinema/SeasonvarScanner.java | 7 ++++++- .../kinosearch3/spider/CinemaDocument.java | 13 +++++++------ .../kinosearch3/cinema/AbstractScannerTest.java | 5 +++++ .../kinosearch3/cinema/OnlinelifeScannerTest.java | 2 -- .../kinosearch3/cinema/SeasonvarScannerTest.java | 2 -- .../kinosearch/kinosearch3/spider/test-spring.xml | 6 +++++- 7 files changed, 28 insertions(+), 13 deletions(-) diff --git a/src/main/java/kinosearch/kinosearch3/cinema/OnlinelifeScanner.java b/src/main/java/kinosearch/kinosearch3/cinema/OnlinelifeScanner.java index 5355c1c..a5c435c 100644 --- a/src/main/java/kinosearch/kinosearch3/cinema/OnlinelifeScanner.java +++ b/src/main/java/kinosearch/kinosearch3/cinema/OnlinelifeScanner.java @@ -59,7 +59,11 @@ public class OnlinelifeScanner implements ScannerCinema { File saveTo = new File(this.saveToDir, url.substring(url.lastIndexOf("/")+1, url.lastIndexOf("."))+".jpg"); this.fileDownloader.addFile(element.attr("src"), saveTo); - CinemaDocument cinemaDocument = new CinemaDocument(saveTo.getAbsolutePath(), this.getName()); + CinemaDocument cinemaDocument = new CinemaDocument(); + cinemaDocument.setTitle(document.getElementsByClass("film_info").get(0).child(0).child(0).text().trim()); + cinemaDocument.setDescription(document.getElementsByClass("film-description").get(0).text()); + cinemaDocument.setFileName(saveTo.getAbsolutePath()); + cinemaDocument.setTypeWarez(this.getName()); repository.save(cinemaDocument); } diff --git a/src/main/java/kinosearch/kinosearch3/cinema/SeasonvarScanner.java b/src/main/java/kinosearch/kinosearch3/cinema/SeasonvarScanner.java index 06a5049..74e2bfe 100644 --- a/src/main/java/kinosearch/kinosearch3/cinema/SeasonvarScanner.java +++ b/src/main/java/kinosearch/kinosearch3/cinema/SeasonvarScanner.java @@ -55,7 +55,12 @@ public class SeasonvarScanner implements ScannerCinema { File saveTo = new File(this.saveTo, url.substring(url.indexOf("/")+1, url.lastIndexOf("."))+".jpg"); this.fileDownloader.addFile(element.attr("src"), saveTo); - CinemaDocument cinemaDocument = new CinemaDocument(saveTo.getAbsolutePath(), this.getName()); + CinemaDocument cinemaDocument = new CinemaDocument(); + String title = document.getElementsByClass("pgs-sinfo-title").get(0).text(); + cinemaDocument.setTitle(title.replaceAll("^Сериал ", "").replaceAll(" онлайн$", "")); + cinemaDocument.setDescription(document.getElementsByAttributeValue("itemprop", "description").get(0).text()); + cinemaDocument.setFileName(saveTo.getAbsolutePath()); + cinemaDocument.setTypeWarez(this.getName()); this.repository.save(cinemaDocument); } } diff --git a/src/main/java/kinosearch/kinosearch3/spider/CinemaDocument.java b/src/main/java/kinosearch/kinosearch3/spider/CinemaDocument.java index bca4875..6d5881c 100644 --- a/src/main/java/kinosearch/kinosearch3/spider/CinemaDocument.java +++ b/src/main/java/kinosearch/kinosearch3/spider/CinemaDocument.java @@ -13,17 +13,18 @@ import org.springframework.data.mongodb.core.mapping.Document; @Document(collection = "cinema") @NoArgsConstructor public class CinemaDocument { - @Id + @Id @Getter private String id; + @Getter @Setter + private String title; + + @Getter @Setter + private String description; + @Getter @Setter private String fileName; @Getter @Setter private String typeWarez; - - public CinemaDocument(String fileName, String typeWarez) { - this.fileName = fileName; - this.typeWarez = typeWarez; - } } diff --git a/src/test/java/kinosearch/kinosearch3/cinema/AbstractScannerTest.java b/src/test/java/kinosearch/kinosearch3/cinema/AbstractScannerTest.java index f1fbe81..7fa2388 100644 --- a/src/test/java/kinosearch/kinosearch3/cinema/AbstractScannerTest.java +++ b/src/test/java/kinosearch/kinosearch3/cinema/AbstractScannerTest.java @@ -24,7 +24,9 @@ public class AbstractScannerTest { private static MongodProcess mongodProcess; @BeforeClass + @SuppressWarnings("deprecation") public static void initBase() throws IOException { + sleep(); IRuntimeConfig runtimeConfig = new RuntimeConfigBuilder() .defaultsWithLogger(Command.MongoD, LoggerFactory.getLogger("Mongod")) .build(); @@ -41,6 +43,9 @@ public class AbstractScannerTest { @AfterClass public static void shutdownBase() { mongodProcess.stop(); + } + + private static void sleep() { try { Thread.sleep(5000L); } catch (InterruptedException e) { diff --git a/src/test/java/kinosearch/kinosearch3/cinema/OnlinelifeScannerTest.java b/src/test/java/kinosearch/kinosearch3/cinema/OnlinelifeScannerTest.java index 9451a7e..ce6ad88 100644 --- a/src/test/java/kinosearch/kinosearch3/cinema/OnlinelifeScannerTest.java +++ b/src/test/java/kinosearch/kinosearch3/cinema/OnlinelifeScannerTest.java @@ -4,8 +4,6 @@ */ package kinosearch.kinosearch3.cinema; -import kinosearch.kinosearch3.cinema.AbstractScannerTest; -import kinosearch.kinosearch3.cinema.OnlinelifeScanner; import kinosearch.kinosearch3.spider.CinemaDocument; import org.junit.*; import org.junit.runner.RunWith; diff --git a/src/test/java/kinosearch/kinosearch3/cinema/SeasonvarScannerTest.java b/src/test/java/kinosearch/kinosearch3/cinema/SeasonvarScannerTest.java index 0f85420..879efc1 100644 --- a/src/test/java/kinosearch/kinosearch3/cinema/SeasonvarScannerTest.java +++ b/src/test/java/kinosearch/kinosearch3/cinema/SeasonvarScannerTest.java @@ -4,8 +4,6 @@ */ package kinosearch.kinosearch3.cinema; -import kinosearch.kinosearch3.cinema.AbstractScannerTest; -import kinosearch.kinosearch3.cinema.SeasonvarScanner; import kinosearch.kinosearch3.spider.CinemaDocument; import org.junit.*; import org.junit.runner.RunWith; diff --git a/src/test/resources/kinosearch/kinosearch3/spider/test-spring.xml b/src/test/resources/kinosearch/kinosearch3/spider/test-spring.xml index f5e6e08..2c7b8dc 100644 --- a/src/test/resources/kinosearch/kinosearch3/spider/test-spring.xml +++ b/src/test/resources/kinosearch/kinosearch3/spider/test-spring.xml @@ -26,7 +26,11 @@ - + + + + +