Archived
0

improve loop search result

This commit is contained in:
iMoHax
2015-08-25 17:44:33 +03:00
parent 7aa8268aa1
commit 9d37544e44
19 changed files with 581 additions and 125 deletions

View File

@@ -1,34 +0,0 @@
package ru.trader.analysis;
import ru.trader.analysis.graph.Edge;
import ru.trader.core.Vendor;
import java.util.List;
import java.util.function.Consumer;
public abstract class AbstractCrawlerSpecification implements CrawlerSpecification {
private final RouteSpecification<Vendor> routeSpecification;
private final Consumer<List<Edge<Vendor>>> onFoundFunc;
private final boolean loop;
protected AbstractCrawlerSpecification(RouteSpecification<Vendor> routeSpecification, Consumer<List<Edge<Vendor>>> onFoundFunc, boolean loop) {
this.routeSpecification = routeSpecification;
this.onFoundFunc = onFoundFunc;
this.loop = loop;
}
protected boolean isLoop() {
return loop;
}
@Override
public RouteSpecification<Vendor> routeSpecification() {
return routeSpecification;
}
@Override
public Consumer<List<Edge<Vendor>>> onFoundFunc() {
return onFoundFunc;
}
}

View File

@@ -1,19 +0,0 @@
package ru.trader.analysis;
import ru.trader.analysis.graph.Edge;
import ru.trader.core.Vendor;
import java.util.List;
import java.util.function.Consumer;
public interface CrawlerSpecification {
public double computeWeight(VendorsCrawler.VendorsEdge edge);
public double computeWeight(VendorsCrawler.VendorsTraversalEntry entry);
public RouteSpecification<Vendor> routeSpecification();
public Consumer<List<Edge<Vendor>>> onFoundFunc();
}

View File

@@ -7,7 +7,7 @@ import java.util.Iterator;
import java.util.List; import java.util.List;
import java.util.function.Consumer; import java.util.function.Consumer;
public class CrawlerSpecificationByProfit extends AbstractCrawlerSpecification { public class CrawlerSpecificationByProfit extends SimpleCrawlerSpecification<Vendor> implements VendorsCrawlerSpecification {
public CrawlerSpecificationByProfit(Consumer<List<Edge<Vendor>>> onFoundFunc) { public CrawlerSpecificationByProfit(Consumer<List<Edge<Vendor>>> onFoundFunc) {
super(null, onFoundFunc, false); super(null, onFoundFunc, false);

View File

@@ -7,7 +7,7 @@ import java.util.Iterator;
import java.util.List; import java.util.List;
import java.util.function.Consumer; import java.util.function.Consumer;
public class CrawlerSpecificationByTime extends AbstractCrawlerSpecification { public class CrawlerSpecificationByTime extends SimpleCrawlerSpecification<Vendor> implements VendorsCrawlerSpecification {
public CrawlerSpecificationByTime(Consumer<List<Edge<Vendor>>> onFoundFunc) { public CrawlerSpecificationByTime(Consumer<List<Edge<Vendor>>> onFoundFunc) {
super(null, onFoundFunc, false); super(null, onFoundFunc, false);
} }

View File

@@ -14,6 +14,7 @@ public class CrawlerSpecificator {
private final List<Vendor> containsAny; private final List<Vendor> containsAny;
private final List<Vendor> all; private final List<Vendor> all;
private final Collection<Offer> offers; private final Collection<Offer> offers;
private int groupCount;
private boolean byTime; private boolean byTime;
public CrawlerSpecificator() { public CrawlerSpecificator() {
@@ -64,7 +65,11 @@ public class CrawlerSpecificator {
this.offers.addAll(offers); this.offers.addAll(offers);
} }
public CrawlerSpecification build(Consumer<List<Edge<Vendor>>> onFoundFunc, RouteSpecification<Vendor> andSpec, boolean loop){ public void setGroupCount(int groupCount) {
this.groupCount = groupCount;
}
public VendorsCrawlerSpecification build(Consumer<List<Edge<Vendor>>> onFoundFunc, RouteSpecification<Vendor> andSpec, boolean loop){
RouteSpecification<Vendor> spec; RouteSpecification<Vendor> spec;
RouteSpecification<Vendor> res = null; RouteSpecification<Vendor> res = null;
if (!all.isEmpty()){ if (!all.isEmpty()){
@@ -94,13 +99,17 @@ public class CrawlerSpecificator {
res = andSpec; res = andSpec;
} }
} }
SimpleCrawlerSpecification crawlerSpecification;
if (byTime){ if (byTime){
return new CrawlerSpecificationByTime(res, onFoundFunc, loop); crawlerSpecification = new CrawlerSpecificationByTime(res, onFoundFunc, loop);
} else {
crawlerSpecification = new CrawlerSpecificationByProfit(res, onFoundFunc, loop);
} }
return new CrawlerSpecificationByProfit(res, onFoundFunc, loop); crawlerSpecification.setGroupCount(groupCount);
return (VendorsCrawlerSpecification) crawlerSpecification;
} }
public CrawlerSpecification build(Consumer<List<Edge<Vendor>>> onFoundFunc){ public VendorsCrawlerSpecification build(Consumer<List<Edge<Vendor>>> onFoundFunc){
return build(onFoundFunc, null, false); return build(onFoundFunc, null, false);
} }

View File

@@ -0,0 +1,310 @@
package ru.trader.analysis;
import org.jetbrains.annotations.NotNull;
import java.util.*;
import java.util.function.Function;
public class GroupLimitedQueue<E> implements Queue<E> {
private final Comparator<? super E> comparator;
private final Function<E, Object> groupGetter;
private final ArrayList<QueueWrapper> queues;
private final int limit;
private boolean sorted = false;
public GroupLimitedQueue(int limit, Function<E, Object> groupGetter) {
this(limit, null, groupGetter);
}
public GroupLimitedQueue(int limit, Comparator<? super E> comparator, Function<E, Object> groupGetter) {
this.limit = limit;
this.comparator = comparator;
this.groupGetter = groupGetter;
queues = new ArrayList<>(10);
}
private LimitedQueue<E> getGroup(Object group, boolean add){
for (QueueWrapper q : queues) {
if (q.group.equals(group)){
return q.queue;
}
}
if (add) {
LimitedQueue<E> queue = new LimitedQueue<>(limit, comparator);
queues.add(new QueueWrapper(group, queue));
sorted = false;
return queue;
} else {
return null;
}
}
private LimitedQueue<E> getGroup(int groupIndex){
return queues.get(groupIndex).queue;
}
private boolean add(Object group, E element){
LimitedQueue<E> queue = getGroup(group, true);
boolean res = queue.add(element);
if (res) sorted = false;
return res;
}
private boolean remove(Object group, E element){
LimitedQueue<E> queue = getGroup(group, false);
if (queue == null) return false;
boolean res = queue.remove(element);
if (res) sorted = false;
return res;
}
private E remove(int groupIndex, int elementIndex){
LimitedQueue<E> queue = getGroup(groupIndex);
sorted = false;
return queue.remove(elementIndex);
}
private E get(int groupIndex, int elementIndex){
LimitedQueue<E> queue = getGroup(groupIndex);
return queue.get(elementIndex);
}
@Override
public boolean add(E element) {
return add(groupGetter.apply(element), element);
}
@Override
public boolean offer(E element) {
return add(element);
}
@Override
public E remove() {
sort();
return remove(0, 0);
}
@SuppressWarnings("unchecked")
@Override
public boolean remove(Object element) {
if (element == null){
boolean res = false;
for (QueueWrapper q : queues) {
res = q.queue.remove(element) || res;
}
return res;
} else {
E e = (E) element;
LimitedQueue<E> queue = getGroup(groupGetter.apply(e), false);
return queue != null && remove(queue, e);
}
}
@Override
public E poll() {
if (isEmpty()) return null;
sort();
return remove(0, 0);
}
@Override
public E element() {
sort();
return get(0, 0);
}
@Override
public E peek() {
if (isEmpty()) return null;
sort();
return get(0, 0);
}
@Override
public int size() {
int size = 0;
for (QueueWrapper q : queues) {
size += q.queue.size();
}
return size;
}
@Override
public boolean isEmpty() {
return size() == 0;
}
@SuppressWarnings("unchecked")
@Override
public boolean contains(Object o) {
if (o == null){
for (QueueWrapper q : queues) {
if (q.queue.contains(o)){
return true;
}
}
} else {
LimitedQueue<E> queue = getGroup(groupGetter.apply((E) o), false);
return queue != null && queue.contains(o);
}
return false;
}
@NotNull
@Override
public Iterator<E> iterator() {
return new Iterator<E>() {
private final Iterator<QueueWrapper> qIter = queues.iterator();
private Iterator<E> iterator;
private E next;
{
nextQueue();
}
private void nextQueue(){
if (qIter.hasNext()) {
iterator = qIter.next().queue.iterator();
nextEntry();
} else {
next = null;
}
}
private void nextEntry(){
if (iterator.hasNext()){
next = iterator.next();
} else {
next = null;
}
}
@Override
public boolean hasNext() {
return next != null;
}
@Override
public E next() {
E res = next;
if (iterator.hasNext()){
nextEntry();
} else {
nextQueue();
}
return res;
}
};
}
@NotNull
@Override
public Object[] toArray() {
Object[] r = new Object[size()];
int index = 0;
for (QueueWrapper q : queues) {
Object[] a = q.queue.toArray();
int s = a.length;
System.arraycopy(a, 0, r, index, s);
index += s;
}
return r;
}
@NotNull
@SuppressWarnings("unchecked")
@Override
public <T> T[] toArray(@NotNull T[] a) {
int size = size();
T[] r = a.length >= size ? a : (T[])java.lang.reflect.Array.newInstance(a.getClass().getComponentType(), size);
int index = 0;
for (QueueWrapper q : queues) {
T[] qa = (T[]) q.queue.toArray();
int s = qa.length;
System.arraycopy(qa, 0, r, index, s);
index += s;
}
return r;
}
@Override
public boolean containsAll(@NotNull Collection<?> c) {
for (Object element : c) {
if (!contains(element)){
return false;
}
}
return true;
}
@SuppressWarnings("unchecked")
@Override
public boolean addAll(@NotNull Collection<? extends E> c) {
boolean modified = false;
if (c instanceof GroupLimitedQueue){
GroupLimitedQueue<E> groupQueue = (GroupLimitedQueue<E>) c;
for (QueueWrapper q : groupQueue.queues) {
LimitedQueue<E> queue = getGroup(q.group, true);
modified = queue.addAll(q.queue) || modified;
}
sorted = sorted && !modified;
} else {
for (E element : c) {
modified = modified || add(element);
}
}
return modified;
}
@Override
public boolean removeAll(@NotNull Collection<?> c) {
boolean modified = false;
for (QueueWrapper q : queues) {
modified = q.queue.removeAll(c) || modified;
}
sorted = sorted && !modified;
return modified;
}
@Override
public boolean retainAll(@NotNull Collection<?> c) {
boolean modified = false;
for (QueueWrapper q : queues) {
modified = q.queue.retainAll(c) || modified;
}
sorted = sorted && !modified;
return modified;
}
@Override
public void clear() {
sorted = false;
queues.clear();
}
public void sort(){
if (sorted || comparator == null) return;
for (QueueWrapper q : queues) {
q.queue.sort();
}
queues.sort((q1, q2) -> {
E e1 = q1.queue.peek();
E e2 = q2.queue.peek();
if (e1 == null || e2 == null){
return e1 == e2 ? 0 : e1 == null ? 1 : -1;
}
return comparator.compare(e1, e2);
});
sorted = true;
}
private class QueueWrapper {
private final Object group;
private final LimitedQueue<E> queue;
private QueueWrapper(Object group, LimitedQueue<E> queue) {
this.group = group;
this.queue = queue;
}
}
}

View File

@@ -5,7 +5,7 @@ import ru.trader.analysis.graph.Traversal;
import java.util.Optional; import java.util.Optional;
public class LoopRouteSpecification<T> implements RouteSpecification<T> { public class LoopRouteSpecification<T> implements MutableRouteSpecification<T> {
private final boolean unique; private final boolean unique;
public LoopRouteSpecification(boolean unique) { public LoopRouteSpecification(boolean unique) {
@@ -27,30 +27,27 @@ public class LoopRouteSpecification<T> implements RouteSpecification<T> {
@Override @Override
public boolean specified(Edge<T> edge, Traversal<T> entry) { public boolean specified(Edge<T> edge, Traversal<T> entry) {
return check(edge, entry, false); return check(edge, entry);
} }
@Override @Override
public boolean updateSpecified(Edge<T> edge, Traversal<T> entry) { public void update(Traversal<T> entry) {
return check(edge, entry, true); setSkip(entry);
} }
private boolean check(Edge<T> edge, Traversal<T> entry, boolean update) { private boolean check(Edge<T> edge, Traversal<T> entry) {
Optional<Traversal<T>> head = entry.getHead(); Optional<Traversal<T>> head = entry.getHead();
if (!head.isPresent() || head.get().getEdge() == null) return false; if (!head.isPresent() || head.get().getEdge() == null) return false;
Traversal<T> start = getStart(head.get()); Traversal<T> start = getStart(head.get());
boolean found = edge.isConnect(start.getTarget().getEntry()); boolean found = edge.isConnect(start.getTarget().getEntry());
if (found && unique){ if (found && unique){
found = !start.isSkipped(); found = !start.isSkipped();
if (update){
start.setSkipped(true);
setSkip(entry);
}
} }
return found; return found;
} }
private void setSkip(Traversal<T> entry) { private void setSkip(Traversal<T> entry) {
if (entry.isSkipped()) return;
Traversal<T> curr = entry; Traversal<T> curr = entry;
Optional<Traversal<T>> head = entry.getHead(); Optional<Traversal<T>> head = entry.getHead();
while (head.isPresent()) { while (head.isPresent()) {

View File

@@ -0,0 +1,10 @@
package ru.trader.analysis;
public interface MutableRouteSpecification<T> extends RouteSpecification<T> {
@Override
public default boolean updateMutated(){return true;}
@Override
public default boolean mutable(){return true;}
}

View File

@@ -49,7 +49,9 @@ public class RouteSearcher {
graph.build(source, places); graph.build(source, places);
LOG.trace("Graph is builds"); LOG.trace("Graph is builds");
List<List<Edge<Place>>> paths = new ArrayList<>(); List<List<Edge<Place>>> paths = new ArrayList<>();
Crawler<Place> crawler = specification != null ? new CCrawler<>(graph, specification, paths::add, callback) : new CCrawler<>(graph, paths::add, callback); Crawler<Place> crawler = specification != null ?
new CCrawler<>(graph, new SimpleCrawlerSpecification<>(specification, paths::add), callback) :
new CCrawler<>(graph, paths::add, callback);
crawler.setMaxSize(profile.getJumps()); crawler.setMaxSize(profile.getJumps());
if (profile.getPathPriority() == Profile.PATH_PRIORITY.FAST){ if (profile.getPathPriority() == Profile.PATH_PRIORITY.FAST){
crawler.findFast(target, count); crawler.findFast(target, count);
@@ -114,9 +116,10 @@ public class RouteSearcher {
vGraph.build(source, vendors); vGraph.build(source, vendors);
LOG.trace("Graph is builds"); LOG.trace("Graph is builds");
RouteCollector collector = new RouteCollector(); RouteCollector collector = new RouteCollector();
specificator.setGroupCount(vendors.size());
Crawler<Vendor> crawler = vGraph.crawler(specificator.build(collector::add, new LoopRouteSpecification<>(true), true), callback); Crawler<Vendor> crawler = vGraph.crawler(specificator.build(collector::add, new LoopRouteSpecification<>(true), true), callback);
crawler.setMaxSize(scorer.getProfile().getLands()); crawler.setMaxSize(scorer.getProfile().getLands());
crawler.findMin(source, count); crawler.findMin(source, vendors.size());
crawler = vGraph.crawler(specificator.build(collector::add, new RouteSpecificationByTarget<>(source), false), callback); crawler = vGraph.crawler(specificator.build(collector::add, new RouteSpecificationByTarget<>(source), false), callback);
crawler.setMaxSize(scorer.getProfile().getLands()); crawler.setMaxSize(scorer.getProfile().getLands());
crawler.findMin(source, 1); crawler.findMin(source, 1);

View File

@@ -6,16 +6,57 @@ import ru.trader.analysis.graph.Traversal;
public interface RouteSpecification<T> { public interface RouteSpecification<T> {
public boolean specified(Edge<T> edge, Traversal<T> entry); public boolean specified(Edge<T> edge, Traversal<T> entry);
public default boolean updateSpecified(Edge<T> edge, Traversal<T> entry){ public default boolean updateMutated(){return false;}
return specified(edge, entry); public default boolean mutable(){return false;}
} public default void update(Traversal<T> entry){}
default RouteSpecification<T> and(final RouteSpecification<T> other){ default RouteSpecification<T> and(final RouteSpecification<T> other){
return (edge, entry) -> RouteSpecification.this.specified(edge, entry) && other.specified(edge, entry); return new RouteSpecification<T>() {
@Override
public boolean specified(Edge<T> edge, Traversal<T> entry) {
return RouteSpecification.this.specified(edge, entry) && other.specified(edge, entry);
}
@Override
public boolean updateMutated() {
return RouteSpecification.this.updateMutated() || other.updateMutated();
}
@Override
public boolean mutable() {
return RouteSpecification.this.mutable() || other.mutable();
}
@Override
public void update(Traversal<T> entry) {
RouteSpecification.this.update(entry);
other.update(entry);
}
};
} }
default RouteSpecification<T> or(final RouteSpecification<T> other){ default RouteSpecification<T> or(final RouteSpecification<T> other){
return (edge, entry) -> RouteSpecification.this.specified(edge, entry) || other.specified(edge, entry); return new RouteSpecification<T>() {
@Override
public boolean specified(Edge<T> edge, Traversal<T> entry) {
return RouteSpecification.this.specified(edge, entry) || other.specified(edge, entry);
}
@Override
public boolean updateMutated() {
return RouteSpecification.this.updateMutated() || other.updateMutated();
}
@Override
public boolean mutable() {
return RouteSpecification.this.mutable() || other.mutable();
}
@Override
public void update(Traversal<T> entry) {
RouteSpecification.this.update(entry);
other.update(entry);
}
};
} }
default RouteSpecification<T> negate(){ default RouteSpecification<T> negate(){

View File

@@ -0,0 +1,78 @@
package ru.trader.analysis;
import ru.trader.analysis.graph.CrawlerSpecification;
import ru.trader.analysis.graph.Edge;
import ru.trader.analysis.graph.Traversal;
import java.util.List;
import java.util.Optional;
import java.util.function.Consumer;
import java.util.function.Function;
public class SimpleCrawlerSpecification<T> implements CrawlerSpecification<T> {
private final RouteSpecification<T> routeSpecification;
private final Consumer<List<Edge<T>>> onFoundFunc;
private final boolean loop;
private int groupCount;
public SimpleCrawlerSpecification(Consumer<List<Edge<T>>> onFoundFunc) {
this(null, onFoundFunc, false);
}
public SimpleCrawlerSpecification(RouteSpecification<T> routeSpecification, Consumer<List<Edge<T>>> onFoundFunc) {
this(routeSpecification, onFoundFunc, false);
}
public SimpleCrawlerSpecification(RouteSpecification<T> routeSpecification, Consumer<List<Edge<T>>> onFoundFunc, boolean loop) {
this.routeSpecification = routeSpecification;
this.onFoundFunc = onFoundFunc;
this.loop = loop;
}
protected boolean isLoop() {
return loop;
}
public void setGroupCount(int groupCount) {
this.groupCount = groupCount;
}
@Override
public RouteSpecification<T> routeSpecification() {
return routeSpecification;
}
@Override
public Consumer<List<Edge<T>>> onFoundFunc() {
return onFoundFunc;
}
public Function<Traversal<T>, Object> getQueueGroupGetter(){
return e -> {
Traversal<T> curr = e;
Traversal<T> target = e;
Optional<Traversal<T>> head = e.getHead();
while (head.isPresent()) {
target = curr;
curr = head.get();
head = curr.getHead();
}
return target.getTarget();
};
}
@Override
public Function<Traversal<T>, Object> getGroupGetter(boolean target) {
if (target){
return Traversal::getTarget;
} else {
return getQueueGroupGetter();
}
}
@Override
public int getGroupCount() {
return groupCount;
}
}

View File

@@ -12,10 +12,10 @@ import java.util.stream.Collectors;
public class VendorsCrawler extends Crawler<Vendor> { public class VendorsCrawler extends Crawler<Vendor> {
private double startFuel; private double startFuel;
private double startBalance; private double startBalance;
private final CrawlerSpecification specification; private final VendorsCrawlerSpecification specification;
public VendorsCrawler(VendorsGraph graph, CrawlerSpecification specification, AnalysisCallBack callback) { public VendorsCrawler(VendorsGraph graph, VendorsCrawlerSpecification specification, AnalysisCallBack callback) {
super(graph, specification.routeSpecification(), specification.onFoundFunc(), callback); super(graph, specification, callback);
this.specification = specification; this.specification = specification;
startFuel = graph.getProfile().getShip().getTank(); startFuel = graph.getProfile().getShip().getTank();
startBalance = graph.getProfile().getBalance(); startBalance = graph.getProfile().getBalance();

View File

@@ -0,0 +1,12 @@
package ru.trader.analysis;
import ru.trader.analysis.graph.CrawlerSpecification;
import ru.trader.core.Vendor;
public interface VendorsCrawlerSpecification extends CrawlerSpecification<Vendor> {
public double computeWeight(VendorsCrawler.VendorsEdge edge);
public double computeWeight(VendorsCrawler.VendorsTraversalEntry entry);
}

View File

@@ -32,7 +32,7 @@ public class VendorsGraph extends ConnectibleGraph<Vendor> {
return new VendorsCrawler(this, new CrawlerSpecificationByTime(onFoundFunc), callback); return new VendorsCrawler(this, new CrawlerSpecificationByTime(onFoundFunc), callback);
} }
public VendorsCrawler crawler(CrawlerSpecification specification, AnalysisCallBack callback){ public VendorsCrawler crawler(VendorsCrawlerSpecification specification, AnalysisCallBack callback){
return new VendorsCrawler(this, specification, callback); return new VendorsCrawler(this, specification, callback);
} }

View File

@@ -21,8 +21,8 @@ public class CCrawler<T extends Connectable<T>> extends Crawler<T> {
init(); init();
} }
public CCrawler(ConnectibleGraph<T> graph, RouteSpecification<T> specification, Consumer<List<Edge<T>>> onFoundFunc, AnalysisCallBack callback) { public CCrawler(ConnectibleGraph<T> graph, CrawlerSpecification<T> specification, AnalysisCallBack callback) {
super(graph, specification, onFoundFunc, callback); super(graph, specification, callback);
init(); init();
} }

View File

@@ -3,9 +3,7 @@ package ru.trader.analysis.graph;
import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.NotNull;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import ru.trader.analysis.AnalysisCallBack; import ru.trader.analysis.*;
import ru.trader.analysis.LimitedQueue;
import ru.trader.analysis.RouteSpecification;
import java.util.*; import java.util.*;
import java.util.concurrent.ForkJoinPool; import java.util.concurrent.ForkJoinPool;
@@ -20,22 +18,22 @@ public class Crawler<T> {
protected final Graph<T> graph; protected final Graph<T> graph;
protected final CrawlerCallBack callback; protected final CrawlerCallBack callback;
private final Consumer<List<Edge<T>>> onFoundFunc;
private final RouteSpecification<T> specification; private final RouteSpecification<T> specification;
private final CrawlerSpecification<T> crawlerSpecification;
private T target; private T target;
private int maxSize; private int maxSize;
public Crawler(Graph<T> graph, Consumer<List<Edge<T>>> onFoundFunc, AnalysisCallBack callback) { public Crawler(Graph<T> graph, Consumer<List<Edge<T>>> onFoundFunc, AnalysisCallBack callback) {
this(graph, null, onFoundFunc, callback); this(graph, new SimpleCrawlerSpecification<>(onFoundFunc), callback);
} }
public Crawler(Graph<T> graph, RouteSpecification<T> specification, Consumer<List<Edge<T>>> onFoundFunc, AnalysisCallBack callback) { public Crawler(Graph<T> graph, CrawlerSpecification<T> specification, AnalysisCallBack callback) {
this.graph = graph; this.graph = graph;
this.callback = new CrawlerCallBack(callback); this.callback = new CrawlerCallBack(callback);
maxSize = graph.getRoot().getLevel(); maxSize = graph.getRoot().getLevel();
this.onFoundFunc = onFoundFunc; crawlerSpecification = specification;
if (specification != null){ if (crawlerSpecification.routeSpecification() != null){
this.specification = specification; this.specification = crawlerSpecification.routeSpecification();
} else { } else {
this.specification = (edge, entry) -> isTarget(edge); this.specification = (edge, entry) -> isTarget(edge);
} }
@@ -64,17 +62,18 @@ public class Crawler<T> {
} }
protected boolean isFound(Edge<T> edge, Traversal<T> head){ protected boolean isFound(Edge<T> edge, Traversal<T> head){
return isFound(edge, head, false); return specification.specified(edge, head);
} }
private boolean isFound(Edge<T> edge, Traversal<T> head, boolean updateStates){ private void updateState(Traversal<T> entry){
return updateStates ? specification.updateSpecified(edge, head) : specification.specified(edge, head); if (specification.mutable()){
specification.update(entry);
}
} }
private void found(List<Edge<T>> res){ private void found(List<Edge<T>> res){
callback.found(); callback.found();
onFoundFunc.accept(res); crawlerSpecification.onFoundFunc().accept(res);
} }
public int getMaxSize() { public int getMaxSize() {
@@ -171,7 +170,8 @@ public class Crawler<T> {
LOG.trace("DFS from {} to {}, deep {}, count {}, entry {}", source, target, deep, count, entry); LOG.trace("DFS from {} to {}, deep {}, count {}, entry {}", source, target, deep, count, entry);
if (deep == source.getLevel()){ if (deep == source.getLevel()){
for (Edge<T> next : entry.getEdges()) { for (Edge<T> next : entry.getEdges()) {
if (isFound(next, entry, true)){ if (isFound(next, entry)){
updateState(entry);
List<Edge<T>> res = getCopyList(entry, next); List<Edge<T>> res = getCopyList(entry, next);
LOG.debug("Last edge found, path {}", res); LOG.debug("Last edge found, path {}", res);
found++; found++;
@@ -222,7 +222,8 @@ public class Crawler<T> {
while (iterator.hasNext()){ while (iterator.hasNext()){
if (callback.isCancel()) break; if (callback.isCancel()) break;
Edge<T> edge = iterator.next(); Edge<T> edge = iterator.next();
if (isFound(edge, entry, true)){ if (isFound(edge, entry)){
updateState(entry);
List<Edge<T>> res = getCopyList(entry, edge); List<Edge<T>> res = getCopyList(entry, edge);
LOG.debug("Last edge found, path {}", res); LOG.debug("Last edge found, path {}", res);
found++; found++;
@@ -259,7 +260,8 @@ public class Crawler<T> {
LOG.trace("Check path entry {}, weight {}", entry, entry.weight); LOG.trace("Check path entry {}, weight {}", entry, entry.weight);
Edge<T> edge = entry.getEdge(); Edge<T> edge = entry.getEdge();
if (edge != null) { if (edge != null) {
if (isFound(edge, entry, true)) { if (isFound(edge, entry)) {
updateState(entry);
List<Edge<T>> res = entry.toEdges(); List<Edge<T>> res = entry.toEdges();
LOG.debug("Path found {}", res); LOG.debug("Path found {}", res);
found++; found++;
@@ -296,15 +298,22 @@ public class Crawler<T> {
LOG.trace("UCS2 from {} to {}, deep {}, count {}", root.vertex, target, deep, count); LOG.trace("UCS2 from {} to {}, deep {}, count {}", root.vertex, target, deep, count);
int found = 0; int found = 0;
double limit = Double.NaN; double limit = Double.NaN;
LimitedQueue<CTEntrySupport> targetsQueue = new LimitedQueue<>(count, Comparator.<CTEntrySupport>naturalOrder()); Queue<CTEntrySupport> targetsQueue;
LimitedQueue<CTEntrySupport> queue = new LimitedQueue<>(count, Comparator.<CTEntrySupport>naturalOrder()); Queue<CTEntrySupport> queue;
if (crawlerSpecification.getGroupCount() > 0){
int routesByGroup = 1 + count / crawlerSpecification.getGroupCount();
targetsQueue = new GroupLimitedQueue<>(routesByGroup, Comparator.<CTEntrySupport>naturalOrder(), e -> crawlerSpecification.getGroupGetter(true).apply(e.entry));
queue = new GroupLimitedQueue<>(routesByGroup * maxSize, Comparator.<CTEntrySupport>naturalOrder(), e -> crawlerSpecification.getGroupGetter(false).apply(e.entry));
} else {
targetsQueue = new LimitedQueue<>(count, Comparator.<CTEntrySupport>naturalOrder());
queue = new LimitedQueue<>(count * maxSize, Comparator.<CTEntrySupport>naturalOrder());
}
root.sort(); root.sort();
queue.add(new CTEntrySupport(root)); queue.add(new CTEntrySupport(root));
while (!(queue.isEmpty() && targetsQueue.isEmpty()) && count > found){ while (!(queue.isEmpty() && targetsQueue.isEmpty()) && count > found){
if (callback.isCancel()) break; if (callback.isCancel()) break;
int alreadyFound = targetsQueue.size();
CTEntrySupport curr = targetsQueue.peek(); CTEntrySupport curr = targetsQueue.peek();
boolean isTarget = curr != null && (queue.isEmpty() || alreadyFound + found >= count || Comparator.<CTEntrySupport>naturalOrder().compare(curr, queue.peek()) <= 0); boolean isTarget = curr != null && (queue.isEmpty() || compareQueue(curr.entry, queue.peek().entry) <= 0);
if (isTarget){ if (isTarget){
targetsQueue.poll(); targetsQueue.poll();
} else { } else {
@@ -312,12 +321,18 @@ public class Crawler<T> {
} }
CostTraversalEntry entry = curr.entry; CostTraversalEntry entry = curr.entry;
LOG.trace("Check path entry {}, weight {}", entry, entry.weight); LOG.trace("Check path entry {}, weight {}", entry, entry.weight);
if (specification.updateMutated() && entry.containsSkipped()){
updateState(entry);
}
if (isTarget) { if (isTarget) {
List<Edge<T>> res = entry.toEdges(); if (!entry.isSkipped()){
LOG.trace("Path found {}", res); updateState(entry);
found++; List<Edge<T>> res = entry.toEdges();
found(res); LOG.trace("Path found {}", res);
if (found >= count) break; found++;
found(res);
if (found >= count) break;
}
CTEntrySupport next = targetsQueue.peek(); CTEntrySupport next = targetsQueue.peek();
limit = next != null ? next.entry.getWeight() : Double.NaN; limit = next != null ? next.entry.getWeight() : Double.NaN;
if (deep > entry.getTarget().getLevel() || entry.size() >= maxSize){ if (deep > entry.getTarget().getLevel() || entry.size() >= maxSize){
@@ -325,12 +340,6 @@ public class Crawler<T> {
continue; continue;
} }
} }
if (alreadyFound + found < count){
LOG.trace("Continue search, limit {}", limit);
} else {
LOG.trace("Already {} found, extracting", alreadyFound);
continue;
}
DFS task = new DFS(curr, deep, count - found, limit); DFS task = new DFS(curr, deep, count - found, limit);
POOL.invoke(task); POOL.invoke(task);
targetsQueue.addAll(task.getTargets()); targetsQueue.addAll(task.getTargets());
@@ -339,6 +348,10 @@ public class Crawler<T> {
return found; return found;
} }
protected int compareQueue(CostTraversalEntry target, CostTraversalEntry queue) {
return target.compareTo(queue);
}
private class CTEntrySupport implements Comparable<CTEntrySupport>, Iterator<Edge<T>>{ private class CTEntrySupport implements Comparable<CTEntrySupport>, Iterator<Edge<T>>{
private final CTEntrySupport parent; private final CTEntrySupport parent;
private Iterator<Edge<T>> iterator; private Iterator<Edge<T>> iterator;
@@ -422,8 +435,14 @@ public class Crawler<T> {
this.deep = deep; this.deep = deep;
this.count = count; this.count = count;
this.limit = limit; this.limit = limit;
queue = new LimitedQueue<>(count, Comparator.<CTEntrySupport>naturalOrder()); if (crawlerSpecification.getGroupCount() > 0){
targets = new LimitedQueue<>(count, Comparator.<CTEntrySupport>naturalOrder()); int routesByGroup = 1 + count / crawlerSpecification.getGroupCount();
targets = new GroupLimitedQueue<>(routesByGroup, Comparator.<CTEntrySupport>naturalOrder(), e -> crawlerSpecification.getGroupGetter(true).apply(e.entry));
queue = new GroupLimitedQueue<>(routesByGroup * maxSize, Comparator.<CTEntrySupport>naturalOrder(), e -> crawlerSpecification.getGroupGetter(false).apply(e.entry));
} else {
targets = new LimitedQueue<>(count, Comparator.<CTEntrySupport>naturalOrder());
queue = new LimitedQueue<>(count * maxSize, Comparator.<CTEntrySupport>naturalOrder());
}
subTasks = new ArrayList<>(THRESHOLD); subTasks = new ArrayList<>(THRESHOLD);
isSubTask = subtask; isSubTask = subtask;
} }
@@ -491,8 +510,8 @@ public class Crawler<T> {
Edge<T> edge = curr.next(); Edge<T> edge = curr.next();
CostTraversalEntry entry = curr.entry; CostTraversalEntry entry = curr.entry;
if (skip()) continue; if (skip()) continue;
LOG.trace("Check edge {}, entry {}, weight {}", edge, entry, entry.weight); LOG.trace("Check edge {}, entry {}, weight {}, curr {}", edge, entry, entry.weight, curr);
boolean isTarget = isFound(edge, entry, true); boolean isTarget = isFound(edge, entry);
boolean canDeep = !entry.getTarget().isSingle() && deep < entry.getTarget().getLevel() && entry.size() < maxSize-1; boolean canDeep = !entry.getTarget().isSingle() && deep < entry.getTarget().getLevel() && entry.size() < maxSize-1;
if (canDeep || isTarget){ if (canDeep || isTarget){
CostTraversalEntry nextEntry = travers(entry, edge); CostTraversalEntry nextEntry = travers(entry, edge);
@@ -508,20 +527,17 @@ public class Crawler<T> {
} else { } else {
if (skip()) continue; if (skip()) continue;
if (!Double.isNaN(limit) && nextEntry.getWeight() >= limit){ if (!Double.isNaN(limit) && nextEntry.getWeight() >= limit){
if (targets.size() < count){ LOG.trace("Not found, limit {}, add entry {} to queue", limit, nextEntry);
LOG.trace("Not found, limit {}, add entry {} to queue", limit, nextEntry); queue.add(curr);
queue.add(curr);
} else {
LOG.trace("Not found, limit {}, don't add entry {} to queue", limit, nextEntry);
}
levelUp(); levelUp();
if (!levelUp()){ if (!levelUp()){
break; break;
} }
} else { } else {
if (!isRoot(curr) && maxSize-nextEntry.size() < SPLIT_SIZE){ if (!isRoot(curr) && maxSize-nextEntry.size() < SPLIT_SIZE){
if (addSubTask(curr)) if (addSubTask(curr)){
levelUp(); levelUp();
}
} }
} }
} }
@@ -635,6 +651,18 @@ public class Crawler<T> {
return skipped; return skipped;
} }
@Override
public boolean containsSkipped() {
if (skipped) return true;
Optional<Traversal<T>> head = getHead();
while (head.isPresent()) {
Traversal<T> curr = head.get();
if (curr.isSkipped()) return true;
head = curr.getHead();
}
return false;
}
protected List<Edge<T>> collect(Collection<Edge<T>> src){ protected List<Edge<T>> collect(Collection<Edge<T>> src){
return new ArrayList<>(src); return new ArrayList<>(src);
} }

View File

@@ -0,0 +1,18 @@
package ru.trader.analysis.graph;
import ru.trader.analysis.RouteSpecification;
import java.util.List;
import java.util.function.Consumer;
import java.util.function.Function;
public interface CrawlerSpecification<T> {
public RouteSpecification<T> routeSpecification();
public Consumer<List<Edge<T>>> onFoundFunc();
public Function<Traversal<T>, Object> getGroupGetter(boolean target);
public int getGroupCount();
}

View File

@@ -11,6 +11,7 @@ public interface Traversal<T> {
void sort(); void sort();
void setSkipped(boolean skipped); void setSkipped(boolean skipped);
boolean isSkipped(); boolean isSkipped();
boolean containsSkipped();
default boolean isConnect(T target){ default boolean isConnect(T target){
Edge<T> edge = getEdge(); Edge<T> edge = getEdge();

View File

@@ -228,7 +228,9 @@ public class VendorsGraphTest extends Assert {
vGraph.build(cabreraDock, fWorld.getMarkets(true).collect(Collectors.toList())); vGraph.build(cabreraDock, fWorld.getMarkets(true).collect(Collectors.toList()));
LOG.info("Search"); LOG.info("Search");
SimpleCollector<Vendor> paths = new SimpleCollector<>(); SimpleCollector<Vendor> paths = new SimpleCollector<>();
Crawler<Vendor> crawler = vGraph.crawler(new CrawlerSpecificationByProfit(new LoopRouteSpecification<>(true), paths::add, true), new AnalysisCallBack()); CrawlerSpecificationByProfit specification = new CrawlerSpecificationByProfit(new LoopRouteSpecification<>(true), paths::add, true);
specification.setGroupCount(60);
Crawler<Vendor> crawler = vGraph.crawler(specification, new AnalysisCallBack());
crawler.findMin(cabreraDock, 100); crawler.findMin(cabreraDock, 100);
assertEquals(60, paths.get().size()); assertEquals(60, paths.get().size());
Collection<Vendor> vendors = new ArrayList<>(60); Collection<Vendor> vendors = new ArrayList<>(60);