Archived
0

implement crawler specificator

This commit is contained in:
iMoHax
2015-08-19 14:46:47 +03:00
parent b7c2ad454a
commit 3bc74482db
2 changed files with 135 additions and 8 deletions

View File

@@ -0,0 +1,107 @@
package ru.trader.analysis;
import ru.trader.analysis.graph.Edge;
import ru.trader.core.Offer;
import ru.trader.core.Vendor;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.function.Consumer;
public class CrawlerSpecificator {
private final List<Vendor> any;
private final List<Vendor> containsAny;
private final List<Vendor> all;
private final Collection<Offer> offers;
private boolean byTime;
public CrawlerSpecificator() {
any = new ArrayList<>();
all = new ArrayList<>();
containsAny = new ArrayList<>();
offers = new ArrayList<>();
byTime = false;
}
public void setByTime(boolean byTime){
this.byTime = byTime;
}
public void all(Collection<Vendor> vendors){
all.addAll(vendors);
}
public void any(Collection<Vendor> vendors){
containsAny.addAll(vendors);
}
public void target(Vendor vendor){
any.add(vendor);
}
public void targetAny(Collection<Vendor> vendors){
any.addAll(vendors);
}
public void add(Vendor vendor, boolean required){
if (required){
all.add(vendor);
} else {
any.add(vendor);
}
}
public void remove(Vendor vendor, boolean required){
if (required){
all.remove(vendor);
} else {
any.remove(vendor);
}
}
public void buy(Collection<Offer> offers){
this.offers.addAll(offers);
}
public CrawlerSpecification build(Consumer<List<Edge<Vendor>>> onFoundFunc, RouteSpecification<Vendor> andSpec){
RouteSpecification<Vendor> spec;
RouteSpecification<Vendor> res = null;
if (!all.isEmpty()){
spec = all.size() > 1 ? RouteSpecificationByTargets.all(all) : new RouteSpecificationByTarget<>(all.get(0));
res = spec;
}
if (!any.isEmpty()){
spec = any.size() > 1 ? RouteSpecificationByTargets.any(any) : new RouteSpecificationByTarget<>(any.get(0));
if (res != null){
res.and(spec);
} else {
res = spec;
}
}
if (!containsAny.isEmpty()){
spec = RouteSpecificationByTargets.containAny(containsAny);
if (res != null){
res.and(spec);
} else {
res = spec;
}
}
if (andSpec != null){
if (res != null){
res.and(andSpec);
} else {
res = andSpec;
}
}
if (byTime){
return new CrawlerSpecificationByTime(res, onFoundFunc);
}
return new CrawlerSpecificationByProfit(res, onFoundFunc);
}
public CrawlerSpecification build(Consumer<List<Edge<Vendor>>> onFoundFunc){
return build(onFoundFunc, null);
}
}

View File

@@ -66,16 +66,19 @@ public class RouteSearcher {
public List<Route> getRoutes(Collection<Vendor> fVendors, Collection<Vendor> toVendors, Collection<Vendor> vendors){
List<Route> res = new LimitedQueue<>(scorer.getProfile().getRoutesCount());
int count = (int) Math.ceil(scorer.getProfile().getRoutesCount() / fVendors.size());
RouteSpecification<Vendor> specification = RouteSpecificationByTargets.any(toVendors);
CrawlerSpecificator specificator = new CrawlerSpecificator();
specificator.any(toVendors);
for (Vendor fromVendor : fVendors) {
Collection<Route> routes = search(fromVendor, fromVendor, vendors, count, specification);
Collection<Route> routes = search(fromVendor, fromVendor, vendors, count, specificator);
res.addAll(routes);
}
return res;
}
public List<Route> getRoutes(Vendor from, Collection<Vendor> vendors){
return search(from, from, vendors, scorer.getProfile().getRoutesCount(), RouteSpecificationByTargets.any(vendors));
CrawlerSpecificator specificator = new CrawlerSpecificator();
specificator.any(vendors);
return search(from, from, vendors, scorer.getProfile().getRoutesCount(), specificator);
}
public List<Route> getRoutes(Vendor from, Vendor to, Collection<Vendor> vendors){
@@ -83,24 +86,41 @@ public class RouteSearcher {
}
public List<Route> getRoutes(Vendor source, Vendor target, Collection<Vendor> vendors, int count){
return search(source, target, vendors, count, null);
return search(source, target, vendors, count, new CrawlerSpecificator());
}
private List<Route> search(Vendor source, Vendor target, Collection<Vendor> vendors, int count, RouteSpecification<Vendor> specification){
public List<Route> getLoops(Vendor source, Collection<Vendor> vendors, int count){
return searchLoops(source, vendors, count, new CrawlerSpecificator());
}
public List<Route> search(Vendor source, Vendor target, Collection<Vendor> vendors, int count, CrawlerSpecificator specificator){
LOG.trace("Start search route from {} to {}", source, target);
VendorsGraph vGraph = new VendorsGraph(scorer, callback);
LOG.trace("Build vendors graph");
vGraph.build(source, vendors);
LOG.trace("Graph is builds");
RouteCollector collector = new RouteCollector();
Crawler<Vendor> crawler = specification != null ? vGraph.crawler(specification, collector::add, callback) : vGraph.crawler(collector::add, callback);
Crawler<Vendor> crawler = vGraph.crawler(specificator.build(collector::add), callback);
crawler.setMaxSize(scorer.getProfile().getLands());
crawler.findMin(target, count);
return collector.get();
}
public List<Route> getLoops(Vendor source, Collection<Vendor> vendors, int count){
return search(source, source, vendors, count, new LoopRouteSpecification<Vendor>(true));
public List<Route> searchLoops(Vendor source, Collection<Vendor> vendors, int count, CrawlerSpecificator specificator){
LOG.trace("Start search loops from {}", source);
VendorsGraph vGraph = new VendorsGraph(scorer, callback);
LOG.trace("Build vendors graph");
vGraph.build(source, vendors);
LOG.trace("Graph is builds");
RouteCollector collector = new RouteCollector();
Crawler<Vendor> crawler = vGraph.crawler(specificator.build(collector::add, new LoopRouteSpecification<>(true)), callback);
crawler.setMaxSize(scorer.getProfile().getLands());
crawler.findMin(source, count);
crawler = vGraph.crawler(specificator.build(collector::add, new RouteSpecificationByTarget<>(source)), callback);
crawler.setMaxSize(scorer.getProfile().getLands());
crawler.findMin(source, 1);
return collector.get();
}
private class RouteCollector {