From 9d37544e4490fa40d3079d92f6c8fb73fcbf57cf Mon Sep 17 00:00:00 2001 From: iMoHax Date: Tue, 25 Aug 2015 17:44:33 +0300 Subject: [PATCH] improve loop search result --- .../AbstractCrawlerSpecification.java | 34 -- .../trader/analysis/CrawlerSpecification.java | 19 -- .../CrawlerSpecificationByProfit.java | 2 +- .../analysis/CrawlerSpecificationByTime.java | 2 +- .../trader/analysis/CrawlerSpecificator.java | 17 +- .../ru/trader/analysis/GroupLimitedQueue.java | 310 ++++++++++++++++++ .../analysis/LoopRouteSpecification.java | 15 +- .../analysis/MutableRouteSpecification.java | 10 + .../ru/trader/analysis/RouteSearcher.java | 7 +- .../trader/analysis/RouteSpecification.java | 51 ++- .../analysis/SimpleCrawlerSpecification.java | 78 +++++ .../ru/trader/analysis/VendorsCrawler.java | 6 +- .../analysis/VendorsCrawlerSpecification.java | 12 + .../java/ru/trader/analysis/VendorsGraph.java | 2 +- .../ru/trader/analysis/graph/CCrawler.java | 4 +- .../ru/trader/analysis/graph/Crawler.java | 114 ++++--- .../analysis/graph/CrawlerSpecification.java | 18 + .../ru/trader/analysis/graph/Traversal.java | 1 + .../ru/trader/analysis/VendorsGraphTest.java | 4 +- 19 files changed, 581 insertions(+), 125 deletions(-) delete mode 100644 core/src/main/java/ru/trader/analysis/AbstractCrawlerSpecification.java delete mode 100644 core/src/main/java/ru/trader/analysis/CrawlerSpecification.java create mode 100644 core/src/main/java/ru/trader/analysis/GroupLimitedQueue.java create mode 100644 core/src/main/java/ru/trader/analysis/MutableRouteSpecification.java create mode 100644 core/src/main/java/ru/trader/analysis/SimpleCrawlerSpecification.java create mode 100644 core/src/main/java/ru/trader/analysis/VendorsCrawlerSpecification.java create mode 100644 core/src/main/java/ru/trader/analysis/graph/CrawlerSpecification.java diff --git a/core/src/main/java/ru/trader/analysis/AbstractCrawlerSpecification.java b/core/src/main/java/ru/trader/analysis/AbstractCrawlerSpecification.java deleted file mode 100644 index 7c0fcf5..0000000 --- a/core/src/main/java/ru/trader/analysis/AbstractCrawlerSpecification.java +++ /dev/null @@ -1,34 +0,0 @@ -package ru.trader.analysis; - - -import ru.trader.analysis.graph.Edge; -import ru.trader.core.Vendor; - -import java.util.List; -import java.util.function.Consumer; - -public abstract class AbstractCrawlerSpecification implements CrawlerSpecification { - private final RouteSpecification routeSpecification; - private final Consumer>> onFoundFunc; - private final boolean loop; - - protected AbstractCrawlerSpecification(RouteSpecification routeSpecification, Consumer>> onFoundFunc, boolean loop) { - this.routeSpecification = routeSpecification; - this.onFoundFunc = onFoundFunc; - this.loop = loop; - } - - protected boolean isLoop() { - return loop; - } - - @Override - public RouteSpecification routeSpecification() { - return routeSpecification; - } - - @Override - public Consumer>> onFoundFunc() { - return onFoundFunc; - } -} diff --git a/core/src/main/java/ru/trader/analysis/CrawlerSpecification.java b/core/src/main/java/ru/trader/analysis/CrawlerSpecification.java deleted file mode 100644 index 296d403..0000000 --- a/core/src/main/java/ru/trader/analysis/CrawlerSpecification.java +++ /dev/null @@ -1,19 +0,0 @@ -package ru.trader.analysis; - -import ru.trader.analysis.graph.Edge; -import ru.trader.core.Vendor; - -import java.util.List; -import java.util.function.Consumer; - -public interface CrawlerSpecification { - - public double computeWeight(VendorsCrawler.VendorsEdge edge); - - public double computeWeight(VendorsCrawler.VendorsTraversalEntry entry); - - public RouteSpecification routeSpecification(); - - public Consumer>> onFoundFunc(); - -} diff --git a/core/src/main/java/ru/trader/analysis/CrawlerSpecificationByProfit.java b/core/src/main/java/ru/trader/analysis/CrawlerSpecificationByProfit.java index 1ab1935..4591ff2 100644 --- a/core/src/main/java/ru/trader/analysis/CrawlerSpecificationByProfit.java +++ b/core/src/main/java/ru/trader/analysis/CrawlerSpecificationByProfit.java @@ -7,7 +7,7 @@ import java.util.Iterator; import java.util.List; import java.util.function.Consumer; -public class CrawlerSpecificationByProfit extends AbstractCrawlerSpecification { +public class CrawlerSpecificationByProfit extends SimpleCrawlerSpecification implements VendorsCrawlerSpecification { public CrawlerSpecificationByProfit(Consumer>> onFoundFunc) { super(null, onFoundFunc, false); diff --git a/core/src/main/java/ru/trader/analysis/CrawlerSpecificationByTime.java b/core/src/main/java/ru/trader/analysis/CrawlerSpecificationByTime.java index d4ad039..66ef595 100644 --- a/core/src/main/java/ru/trader/analysis/CrawlerSpecificationByTime.java +++ b/core/src/main/java/ru/trader/analysis/CrawlerSpecificationByTime.java @@ -7,7 +7,7 @@ import java.util.Iterator; import java.util.List; import java.util.function.Consumer; -public class CrawlerSpecificationByTime extends AbstractCrawlerSpecification { +public class CrawlerSpecificationByTime extends SimpleCrawlerSpecification implements VendorsCrawlerSpecification { public CrawlerSpecificationByTime(Consumer>> onFoundFunc) { super(null, onFoundFunc, false); } diff --git a/core/src/main/java/ru/trader/analysis/CrawlerSpecificator.java b/core/src/main/java/ru/trader/analysis/CrawlerSpecificator.java index 18cf88a..b1a72c8 100644 --- a/core/src/main/java/ru/trader/analysis/CrawlerSpecificator.java +++ b/core/src/main/java/ru/trader/analysis/CrawlerSpecificator.java @@ -14,6 +14,7 @@ public class CrawlerSpecificator { private final List containsAny; private final List all; private final Collection offers; + private int groupCount; private boolean byTime; public CrawlerSpecificator() { @@ -64,7 +65,11 @@ public class CrawlerSpecificator { this.offers.addAll(offers); } - public CrawlerSpecification build(Consumer>> onFoundFunc, RouteSpecification andSpec, boolean loop){ + public void setGroupCount(int groupCount) { + this.groupCount = groupCount; + } + + public VendorsCrawlerSpecification build(Consumer>> onFoundFunc, RouteSpecification andSpec, boolean loop){ RouteSpecification spec; RouteSpecification res = null; if (!all.isEmpty()){ @@ -94,13 +99,17 @@ public class CrawlerSpecificator { res = andSpec; } } + SimpleCrawlerSpecification crawlerSpecification; if (byTime){ - return new CrawlerSpecificationByTime(res, onFoundFunc, loop); + crawlerSpecification = new CrawlerSpecificationByTime(res, onFoundFunc, loop); + } else { + crawlerSpecification = new CrawlerSpecificationByProfit(res, onFoundFunc, loop); } - return new CrawlerSpecificationByProfit(res, onFoundFunc, loop); + crawlerSpecification.setGroupCount(groupCount); + return (VendorsCrawlerSpecification) crawlerSpecification; } - public CrawlerSpecification build(Consumer>> onFoundFunc){ + public VendorsCrawlerSpecification build(Consumer>> onFoundFunc){ return build(onFoundFunc, null, false); } diff --git a/core/src/main/java/ru/trader/analysis/GroupLimitedQueue.java b/core/src/main/java/ru/trader/analysis/GroupLimitedQueue.java new file mode 100644 index 0000000..775b7ff --- /dev/null +++ b/core/src/main/java/ru/trader/analysis/GroupLimitedQueue.java @@ -0,0 +1,310 @@ +package ru.trader.analysis; + +import org.jetbrains.annotations.NotNull; + +import java.util.*; +import java.util.function.Function; + +public class GroupLimitedQueue implements Queue { + private final Comparator comparator; + private final Function groupGetter; + private final ArrayList queues; + private final int limit; + private boolean sorted = false; + + public GroupLimitedQueue(int limit, Function groupGetter) { + this(limit, null, groupGetter); + } + + public GroupLimitedQueue(int limit, Comparator comparator, Function groupGetter) { + this.limit = limit; + this.comparator = comparator; + this.groupGetter = groupGetter; + queues = new ArrayList<>(10); + } + + private LimitedQueue getGroup(Object group, boolean add){ + for (QueueWrapper q : queues) { + if (q.group.equals(group)){ + return q.queue; + } + } + if (add) { + LimitedQueue queue = new LimitedQueue<>(limit, comparator); + queues.add(new QueueWrapper(group, queue)); + sorted = false; + return queue; + } else { + return null; + } + } + + private LimitedQueue getGroup(int groupIndex){ + return queues.get(groupIndex).queue; + } + + private boolean add(Object group, E element){ + LimitedQueue queue = getGroup(group, true); + boolean res = queue.add(element); + if (res) sorted = false; + return res; + } + + private boolean remove(Object group, E element){ + LimitedQueue queue = getGroup(group, false); + if (queue == null) return false; + boolean res = queue.remove(element); + if (res) sorted = false; + return res; + } + + private E remove(int groupIndex, int elementIndex){ + LimitedQueue queue = getGroup(groupIndex); + sorted = false; + return queue.remove(elementIndex); + } + + private E get(int groupIndex, int elementIndex){ + LimitedQueue queue = getGroup(groupIndex); + return queue.get(elementIndex); + } + + @Override + public boolean add(E element) { + return add(groupGetter.apply(element), element); + } + + @Override + public boolean offer(E element) { + return add(element); + } + + @Override + public E remove() { + sort(); + return remove(0, 0); + } + + @SuppressWarnings("unchecked") + @Override + public boolean remove(Object element) { + if (element == null){ + boolean res = false; + for (QueueWrapper q : queues) { + res = q.queue.remove(element) || res; + } + return res; + } else { + E e = (E) element; + LimitedQueue queue = getGroup(groupGetter.apply(e), false); + return queue != null && remove(queue, e); + } + } + + @Override + public E poll() { + if (isEmpty()) return null; + sort(); + return remove(0, 0); + } + + @Override + public E element() { + sort(); + return get(0, 0); + } + + @Override + public E peek() { + if (isEmpty()) return null; + sort(); + return get(0, 0); + } + + @Override + public int size() { + int size = 0; + for (QueueWrapper q : queues) { + size += q.queue.size(); + } + return size; + } + + @Override + public boolean isEmpty() { + return size() == 0; + } + + @SuppressWarnings("unchecked") + @Override + public boolean contains(Object o) { + if (o == null){ + for (QueueWrapper q : queues) { + if (q.queue.contains(o)){ + return true; + } + } + } else { + LimitedQueue queue = getGroup(groupGetter.apply((E) o), false); + return queue != null && queue.contains(o); + } + return false; + } + + @NotNull + @Override + public Iterator iterator() { + return new Iterator() { + private final Iterator qIter = queues.iterator(); + private Iterator iterator; + private E next; + { + nextQueue(); + } + + private void nextQueue(){ + if (qIter.hasNext()) { + iterator = qIter.next().queue.iterator(); + nextEntry(); + } else { + next = null; + } + } + + private void nextEntry(){ + if (iterator.hasNext()){ + next = iterator.next(); + } else { + next = null; + } + } + + @Override + public boolean hasNext() { + return next != null; + } + + @Override + public E next() { + E res = next; + if (iterator.hasNext()){ + nextEntry(); + } else { + nextQueue(); + } + return res; + } + }; + } + + @NotNull + @Override + public Object[] toArray() { + Object[] r = new Object[size()]; + int index = 0; + for (QueueWrapper q : queues) { + Object[] a = q.queue.toArray(); + int s = a.length; + System.arraycopy(a, 0, r, index, s); + index += s; + } + return r; + } + + @NotNull + @SuppressWarnings("unchecked") + @Override + public T[] toArray(@NotNull T[] a) { + int size = size(); + T[] r = a.length >= size ? a : (T[])java.lang.reflect.Array.newInstance(a.getClass().getComponentType(), size); + int index = 0; + for (QueueWrapper q : queues) { + T[] qa = (T[]) q.queue.toArray(); + int s = qa.length; + System.arraycopy(qa, 0, r, index, s); + index += s; + } + return r; + } + + @Override + public boolean containsAll(@NotNull Collection c) { + for (Object element : c) { + if (!contains(element)){ + return false; + } + } + return true; + } + + @SuppressWarnings("unchecked") + @Override + public boolean addAll(@NotNull Collection c) { + boolean modified = false; + if (c instanceof GroupLimitedQueue){ + GroupLimitedQueue groupQueue = (GroupLimitedQueue) c; + for (QueueWrapper q : groupQueue.queues) { + LimitedQueue queue = getGroup(q.group, true); + modified = queue.addAll(q.queue) || modified; + } + sorted = sorted && !modified; + } else { + for (E element : c) { + modified = modified || add(element); + } + } + return modified; + } + + @Override + public boolean removeAll(@NotNull Collection c) { + boolean modified = false; + for (QueueWrapper q : queues) { + modified = q.queue.removeAll(c) || modified; + } + sorted = sorted && !modified; + return modified; + } + + @Override + public boolean retainAll(@NotNull Collection c) { + boolean modified = false; + for (QueueWrapper q : queues) { + modified = q.queue.retainAll(c) || modified; + } + sorted = sorted && !modified; + return modified; + } + + @Override + public void clear() { + sorted = false; + queues.clear(); + } + + public void sort(){ + if (sorted || comparator == null) return; + for (QueueWrapper q : queues) { + q.queue.sort(); + } + queues.sort((q1, q2) -> { + E e1 = q1.queue.peek(); + E e2 = q2.queue.peek(); + if (e1 == null || e2 == null){ + return e1 == e2 ? 0 : e1 == null ? 1 : -1; + } + return comparator.compare(e1, e2); + }); + sorted = true; + } + + private class QueueWrapper { + private final Object group; + private final LimitedQueue queue; + + private QueueWrapper(Object group, LimitedQueue queue) { + this.group = group; + this.queue = queue; + } + } +} diff --git a/core/src/main/java/ru/trader/analysis/LoopRouteSpecification.java b/core/src/main/java/ru/trader/analysis/LoopRouteSpecification.java index ed51506..3e531e3 100644 --- a/core/src/main/java/ru/trader/analysis/LoopRouteSpecification.java +++ b/core/src/main/java/ru/trader/analysis/LoopRouteSpecification.java @@ -5,7 +5,7 @@ import ru.trader.analysis.graph.Traversal; import java.util.Optional; -public class LoopRouteSpecification implements RouteSpecification { +public class LoopRouteSpecification implements MutableRouteSpecification { private final boolean unique; public LoopRouteSpecification(boolean unique) { @@ -27,30 +27,27 @@ public class LoopRouteSpecification implements RouteSpecification { @Override public boolean specified(Edge edge, Traversal entry) { - return check(edge, entry, false); + return check(edge, entry); } @Override - public boolean updateSpecified(Edge edge, Traversal entry) { - return check(edge, entry, true); + public void update(Traversal entry) { + setSkip(entry); } - private boolean check(Edge edge, Traversal entry, boolean update) { + private boolean check(Edge edge, Traversal entry) { Optional> head = entry.getHead(); if (!head.isPresent() || head.get().getEdge() == null) return false; Traversal start = getStart(head.get()); boolean found = edge.isConnect(start.getTarget().getEntry()); if (found && unique){ found = !start.isSkipped(); - if (update){ - start.setSkipped(true); - setSkip(entry); - } } return found; } private void setSkip(Traversal entry) { + if (entry.isSkipped()) return; Traversal curr = entry; Optional> head = entry.getHead(); while (head.isPresent()) { diff --git a/core/src/main/java/ru/trader/analysis/MutableRouteSpecification.java b/core/src/main/java/ru/trader/analysis/MutableRouteSpecification.java new file mode 100644 index 0000000..e2fe973 --- /dev/null +++ b/core/src/main/java/ru/trader/analysis/MutableRouteSpecification.java @@ -0,0 +1,10 @@ +package ru.trader.analysis; + +public interface MutableRouteSpecification extends RouteSpecification { + + @Override + public default boolean updateMutated(){return true;} + @Override + public default boolean mutable(){return true;} + +} diff --git a/core/src/main/java/ru/trader/analysis/RouteSearcher.java b/core/src/main/java/ru/trader/analysis/RouteSearcher.java index 54e52f0..8550238 100644 --- a/core/src/main/java/ru/trader/analysis/RouteSearcher.java +++ b/core/src/main/java/ru/trader/analysis/RouteSearcher.java @@ -49,7 +49,9 @@ public class RouteSearcher { graph.build(source, places); LOG.trace("Graph is builds"); List>> paths = new ArrayList<>(); - Crawler crawler = specification != null ? new CCrawler<>(graph, specification, paths::add, callback) : new CCrawler<>(graph, paths::add, callback); + Crawler crawler = specification != null ? + new CCrawler<>(graph, new SimpleCrawlerSpecification<>(specification, paths::add), callback) : + new CCrawler<>(graph, paths::add, callback); crawler.setMaxSize(profile.getJumps()); if (profile.getPathPriority() == Profile.PATH_PRIORITY.FAST){ crawler.findFast(target, count); @@ -114,9 +116,10 @@ public class RouteSearcher { vGraph.build(source, vendors); LOG.trace("Graph is builds"); RouteCollector collector = new RouteCollector(); + specificator.setGroupCount(vendors.size()); Crawler crawler = vGraph.crawler(specificator.build(collector::add, new LoopRouteSpecification<>(true), true), callback); crawler.setMaxSize(scorer.getProfile().getLands()); - crawler.findMin(source, count); + crawler.findMin(source, vendors.size()); crawler = vGraph.crawler(specificator.build(collector::add, new RouteSpecificationByTarget<>(source), false), callback); crawler.setMaxSize(scorer.getProfile().getLands()); crawler.findMin(source, 1); diff --git a/core/src/main/java/ru/trader/analysis/RouteSpecification.java b/core/src/main/java/ru/trader/analysis/RouteSpecification.java index 3d4f161..c50a006 100644 --- a/core/src/main/java/ru/trader/analysis/RouteSpecification.java +++ b/core/src/main/java/ru/trader/analysis/RouteSpecification.java @@ -6,16 +6,57 @@ import ru.trader.analysis.graph.Traversal; public interface RouteSpecification { public boolean specified(Edge edge, Traversal entry); - public default boolean updateSpecified(Edge edge, Traversal entry){ - return specified(edge, entry); - } + public default boolean updateMutated(){return false;} + public default boolean mutable(){return false;} + public default void update(Traversal entry){} default RouteSpecification and(final RouteSpecification other){ - return (edge, entry) -> RouteSpecification.this.specified(edge, entry) && other.specified(edge, entry); + return new RouteSpecification() { + @Override + public boolean specified(Edge edge, Traversal entry) { + return RouteSpecification.this.specified(edge, entry) && other.specified(edge, entry); + } + + @Override + public boolean updateMutated() { + return RouteSpecification.this.updateMutated() || other.updateMutated(); + } + + @Override + public boolean mutable() { + return RouteSpecification.this.mutable() || other.mutable(); + } + + @Override + public void update(Traversal entry) { + RouteSpecification.this.update(entry); + other.update(entry); + } + }; } default RouteSpecification or(final RouteSpecification other){ - return (edge, entry) -> RouteSpecification.this.specified(edge, entry) || other.specified(edge, entry); + return new RouteSpecification() { + @Override + public boolean specified(Edge edge, Traversal entry) { + return RouteSpecification.this.specified(edge, entry) || other.specified(edge, entry); + } + + @Override + public boolean updateMutated() { + return RouteSpecification.this.updateMutated() || other.updateMutated(); + } + @Override + public boolean mutable() { + return RouteSpecification.this.mutable() || other.mutable(); + } + + @Override + public void update(Traversal entry) { + RouteSpecification.this.update(entry); + other.update(entry); + } + }; } default RouteSpecification negate(){ diff --git a/core/src/main/java/ru/trader/analysis/SimpleCrawlerSpecification.java b/core/src/main/java/ru/trader/analysis/SimpleCrawlerSpecification.java new file mode 100644 index 0000000..ce691ac --- /dev/null +++ b/core/src/main/java/ru/trader/analysis/SimpleCrawlerSpecification.java @@ -0,0 +1,78 @@ +package ru.trader.analysis; + + +import ru.trader.analysis.graph.CrawlerSpecification; +import ru.trader.analysis.graph.Edge; +import ru.trader.analysis.graph.Traversal; + +import java.util.List; +import java.util.Optional; +import java.util.function.Consumer; +import java.util.function.Function; + +public class SimpleCrawlerSpecification implements CrawlerSpecification { + private final RouteSpecification routeSpecification; + private final Consumer>> onFoundFunc; + private final boolean loop; + private int groupCount; + + public SimpleCrawlerSpecification(Consumer>> onFoundFunc) { + this(null, onFoundFunc, false); + } + + public SimpleCrawlerSpecification(RouteSpecification routeSpecification, Consumer>> onFoundFunc) { + this(routeSpecification, onFoundFunc, false); + } + + public SimpleCrawlerSpecification(RouteSpecification routeSpecification, Consumer>> onFoundFunc, boolean loop) { + this.routeSpecification = routeSpecification; + this.onFoundFunc = onFoundFunc; + this.loop = loop; + } + + protected boolean isLoop() { + return loop; + } + + public void setGroupCount(int groupCount) { + this.groupCount = groupCount; + } + + @Override + public RouteSpecification routeSpecification() { + return routeSpecification; + } + + @Override + public Consumer>> onFoundFunc() { + return onFoundFunc; + } + + public Function, Object> getQueueGroupGetter(){ + return e -> { + Traversal curr = e; + Traversal target = e; + Optional> head = e.getHead(); + while (head.isPresent()) { + target = curr; + curr = head.get(); + head = curr.getHead(); + } + return target.getTarget(); + }; + } + + @Override + public Function, Object> getGroupGetter(boolean target) { + if (target){ + return Traversal::getTarget; + } else { + return getQueueGroupGetter(); + } + } + + @Override + public int getGroupCount() { + return groupCount; + } +} diff --git a/core/src/main/java/ru/trader/analysis/VendorsCrawler.java b/core/src/main/java/ru/trader/analysis/VendorsCrawler.java index f5ba9c1..5a91726 100644 --- a/core/src/main/java/ru/trader/analysis/VendorsCrawler.java +++ b/core/src/main/java/ru/trader/analysis/VendorsCrawler.java @@ -12,10 +12,10 @@ import java.util.stream.Collectors; public class VendorsCrawler extends Crawler { private double startFuel; private double startBalance; - private final CrawlerSpecification specification; + private final VendorsCrawlerSpecification specification; - public VendorsCrawler(VendorsGraph graph, CrawlerSpecification specification, AnalysisCallBack callback) { - super(graph, specification.routeSpecification(), specification.onFoundFunc(), callback); + public VendorsCrawler(VendorsGraph graph, VendorsCrawlerSpecification specification, AnalysisCallBack callback) { + super(graph, specification, callback); this.specification = specification; startFuel = graph.getProfile().getShip().getTank(); startBalance = graph.getProfile().getBalance(); diff --git a/core/src/main/java/ru/trader/analysis/VendorsCrawlerSpecification.java b/core/src/main/java/ru/trader/analysis/VendorsCrawlerSpecification.java new file mode 100644 index 0000000..f9ca193 --- /dev/null +++ b/core/src/main/java/ru/trader/analysis/VendorsCrawlerSpecification.java @@ -0,0 +1,12 @@ +package ru.trader.analysis; + +import ru.trader.analysis.graph.CrawlerSpecification; +import ru.trader.core.Vendor; + +public interface VendorsCrawlerSpecification extends CrawlerSpecification { + + public double computeWeight(VendorsCrawler.VendorsEdge edge); + + public double computeWeight(VendorsCrawler.VendorsTraversalEntry entry); + +} diff --git a/core/src/main/java/ru/trader/analysis/VendorsGraph.java b/core/src/main/java/ru/trader/analysis/VendorsGraph.java index d90d041..f0d6422 100644 --- a/core/src/main/java/ru/trader/analysis/VendorsGraph.java +++ b/core/src/main/java/ru/trader/analysis/VendorsGraph.java @@ -32,7 +32,7 @@ public class VendorsGraph extends ConnectibleGraph { return new VendorsCrawler(this, new CrawlerSpecificationByTime(onFoundFunc), callback); } - public VendorsCrawler crawler(CrawlerSpecification specification, AnalysisCallBack callback){ + public VendorsCrawler crawler(VendorsCrawlerSpecification specification, AnalysisCallBack callback){ return new VendorsCrawler(this, specification, callback); } diff --git a/core/src/main/java/ru/trader/analysis/graph/CCrawler.java b/core/src/main/java/ru/trader/analysis/graph/CCrawler.java index 01bec4a..a02a03f 100644 --- a/core/src/main/java/ru/trader/analysis/graph/CCrawler.java +++ b/core/src/main/java/ru/trader/analysis/graph/CCrawler.java @@ -21,8 +21,8 @@ public class CCrawler> extends Crawler { init(); } - public CCrawler(ConnectibleGraph graph, RouteSpecification specification, Consumer>> onFoundFunc, AnalysisCallBack callback) { - super(graph, specification, onFoundFunc, callback); + public CCrawler(ConnectibleGraph graph, CrawlerSpecification specification, AnalysisCallBack callback) { + super(graph, specification, callback); init(); } diff --git a/core/src/main/java/ru/trader/analysis/graph/Crawler.java b/core/src/main/java/ru/trader/analysis/graph/Crawler.java index 877f83a..31d6298 100644 --- a/core/src/main/java/ru/trader/analysis/graph/Crawler.java +++ b/core/src/main/java/ru/trader/analysis/graph/Crawler.java @@ -3,9 +3,7 @@ package ru.trader.analysis.graph; import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import ru.trader.analysis.AnalysisCallBack; -import ru.trader.analysis.LimitedQueue; -import ru.trader.analysis.RouteSpecification; +import ru.trader.analysis.*; import java.util.*; import java.util.concurrent.ForkJoinPool; @@ -20,22 +18,22 @@ public class Crawler { protected final Graph graph; protected final CrawlerCallBack callback; - private final Consumer>> onFoundFunc; private final RouteSpecification specification; + private final CrawlerSpecification crawlerSpecification; private T target; private int maxSize; public Crawler(Graph graph, Consumer>> onFoundFunc, AnalysisCallBack callback) { - this(graph, null, onFoundFunc, callback); + this(graph, new SimpleCrawlerSpecification<>(onFoundFunc), callback); } - public Crawler(Graph graph, RouteSpecification specification, Consumer>> onFoundFunc, AnalysisCallBack callback) { + public Crawler(Graph graph, CrawlerSpecification specification, AnalysisCallBack callback) { this.graph = graph; this.callback = new CrawlerCallBack(callback); maxSize = graph.getRoot().getLevel(); - this.onFoundFunc = onFoundFunc; - if (specification != null){ - this.specification = specification; + crawlerSpecification = specification; + if (crawlerSpecification.routeSpecification() != null){ + this.specification = crawlerSpecification.routeSpecification(); } else { this.specification = (edge, entry) -> isTarget(edge); } @@ -64,17 +62,18 @@ public class Crawler { } protected boolean isFound(Edge edge, Traversal head){ - return isFound(edge, head, false); + return specification.specified(edge, head); } - private boolean isFound(Edge edge, Traversal head, boolean updateStates){ - return updateStates ? specification.updateSpecified(edge, head) : specification.specified(edge, head); + private void updateState(Traversal entry){ + if (specification.mutable()){ + specification.update(entry); + } } private void found(List> res){ callback.found(); - onFoundFunc.accept(res); - + crawlerSpecification.onFoundFunc().accept(res); } public int getMaxSize() { @@ -171,7 +170,8 @@ public class Crawler { LOG.trace("DFS from {} to {}, deep {}, count {}, entry {}", source, target, deep, count, entry); if (deep == source.getLevel()){ for (Edge next : entry.getEdges()) { - if (isFound(next, entry, true)){ + if (isFound(next, entry)){ + updateState(entry); List> res = getCopyList(entry, next); LOG.debug("Last edge found, path {}", res); found++; @@ -222,7 +222,8 @@ public class Crawler { while (iterator.hasNext()){ if (callback.isCancel()) break; Edge edge = iterator.next(); - if (isFound(edge, entry, true)){ + if (isFound(edge, entry)){ + updateState(entry); List> res = getCopyList(entry, edge); LOG.debug("Last edge found, path {}", res); found++; @@ -259,7 +260,8 @@ public class Crawler { LOG.trace("Check path entry {}, weight {}", entry, entry.weight); Edge edge = entry.getEdge(); if (edge != null) { - if (isFound(edge, entry, true)) { + if (isFound(edge, entry)) { + updateState(entry); List> res = entry.toEdges(); LOG.debug("Path found {}", res); found++; @@ -296,15 +298,22 @@ public class Crawler { LOG.trace("UCS2 from {} to {}, deep {}, count {}", root.vertex, target, deep, count); int found = 0; double limit = Double.NaN; - LimitedQueue targetsQueue = new LimitedQueue<>(count, Comparator.naturalOrder()); - LimitedQueue queue = new LimitedQueue<>(count, Comparator.naturalOrder()); + Queue targetsQueue; + Queue queue; + if (crawlerSpecification.getGroupCount() > 0){ + int routesByGroup = 1 + count / crawlerSpecification.getGroupCount(); + targetsQueue = new GroupLimitedQueue<>(routesByGroup, Comparator.naturalOrder(), e -> crawlerSpecification.getGroupGetter(true).apply(e.entry)); + queue = new GroupLimitedQueue<>(routesByGroup * maxSize, Comparator.naturalOrder(), e -> crawlerSpecification.getGroupGetter(false).apply(e.entry)); + } else { + targetsQueue = new LimitedQueue<>(count, Comparator.naturalOrder()); + queue = new LimitedQueue<>(count * maxSize, Comparator.naturalOrder()); + } root.sort(); queue.add(new CTEntrySupport(root)); while (!(queue.isEmpty() && targetsQueue.isEmpty()) && count > found){ if (callback.isCancel()) break; - int alreadyFound = targetsQueue.size(); CTEntrySupport curr = targetsQueue.peek(); - boolean isTarget = curr != null && (queue.isEmpty() || alreadyFound + found >= count || Comparator.naturalOrder().compare(curr, queue.peek()) <= 0); + boolean isTarget = curr != null && (queue.isEmpty() || compareQueue(curr.entry, queue.peek().entry) <= 0); if (isTarget){ targetsQueue.poll(); } else { @@ -312,12 +321,18 @@ public class Crawler { } CostTraversalEntry entry = curr.entry; LOG.trace("Check path entry {}, weight {}", entry, entry.weight); + if (specification.updateMutated() && entry.containsSkipped()){ + updateState(entry); + } if (isTarget) { - List> res = entry.toEdges(); - LOG.trace("Path found {}", res); - found++; - found(res); - if (found >= count) break; + if (!entry.isSkipped()){ + updateState(entry); + List> res = entry.toEdges(); + LOG.trace("Path found {}", res); + found++; + found(res); + if (found >= count) break; + } CTEntrySupport next = targetsQueue.peek(); limit = next != null ? next.entry.getWeight() : Double.NaN; if (deep > entry.getTarget().getLevel() || entry.size() >= maxSize){ @@ -325,12 +340,6 @@ public class Crawler { continue; } } - if (alreadyFound + found < count){ - LOG.trace("Continue search, limit {}", limit); - } else { - LOG.trace("Already {} found, extracting", alreadyFound); - continue; - } DFS task = new DFS(curr, deep, count - found, limit); POOL.invoke(task); targetsQueue.addAll(task.getTargets()); @@ -339,6 +348,10 @@ public class Crawler { return found; } + protected int compareQueue(CostTraversalEntry target, CostTraversalEntry queue) { + return target.compareTo(queue); + } + private class CTEntrySupport implements Comparable, Iterator>{ private final CTEntrySupport parent; private Iterator> iterator; @@ -422,8 +435,14 @@ public class Crawler { this.deep = deep; this.count = count; this.limit = limit; - queue = new LimitedQueue<>(count, Comparator.naturalOrder()); - targets = new LimitedQueue<>(count, Comparator.naturalOrder()); + if (crawlerSpecification.getGroupCount() > 0){ + int routesByGroup = 1 + count / crawlerSpecification.getGroupCount(); + targets = new GroupLimitedQueue<>(routesByGroup, Comparator.naturalOrder(), e -> crawlerSpecification.getGroupGetter(true).apply(e.entry)); + queue = new GroupLimitedQueue<>(routesByGroup * maxSize, Comparator.naturalOrder(), e -> crawlerSpecification.getGroupGetter(false).apply(e.entry)); + } else { + targets = new LimitedQueue<>(count, Comparator.naturalOrder()); + queue = new LimitedQueue<>(count * maxSize, Comparator.naturalOrder()); + } subTasks = new ArrayList<>(THRESHOLD); isSubTask = subtask; } @@ -491,8 +510,8 @@ public class Crawler { Edge edge = curr.next(); CostTraversalEntry entry = curr.entry; if (skip()) continue; - LOG.trace("Check edge {}, entry {}, weight {}", edge, entry, entry.weight); - boolean isTarget = isFound(edge, entry, true); + LOG.trace("Check edge {}, entry {}, weight {}, curr {}", edge, entry, entry.weight, curr); + boolean isTarget = isFound(edge, entry); boolean canDeep = !entry.getTarget().isSingle() && deep < entry.getTarget().getLevel() && entry.size() < maxSize-1; if (canDeep || isTarget){ CostTraversalEntry nextEntry = travers(entry, edge); @@ -508,20 +527,17 @@ public class Crawler { } else { if (skip()) continue; if (!Double.isNaN(limit) && nextEntry.getWeight() >= limit){ - if (targets.size() < count){ - LOG.trace("Not found, limit {}, add entry {} to queue", limit, nextEntry); - queue.add(curr); - } else { - LOG.trace("Not found, limit {}, don't add entry {} to queue", limit, nextEntry); - } + LOG.trace("Not found, limit {}, add entry {} to queue", limit, nextEntry); + queue.add(curr); levelUp(); if (!levelUp()){ break; } } else { if (!isRoot(curr) && maxSize-nextEntry.size() < SPLIT_SIZE){ - if (addSubTask(curr)) + if (addSubTask(curr)){ levelUp(); + } } } } @@ -635,6 +651,18 @@ public class Crawler { return skipped; } + @Override + public boolean containsSkipped() { + if (skipped) return true; + Optional> head = getHead(); + while (head.isPresent()) { + Traversal curr = head.get(); + if (curr.isSkipped()) return true; + head = curr.getHead(); + } + return false; + } + protected List> collect(Collection> src){ return new ArrayList<>(src); } diff --git a/core/src/main/java/ru/trader/analysis/graph/CrawlerSpecification.java b/core/src/main/java/ru/trader/analysis/graph/CrawlerSpecification.java new file mode 100644 index 0000000..bd2593d --- /dev/null +++ b/core/src/main/java/ru/trader/analysis/graph/CrawlerSpecification.java @@ -0,0 +1,18 @@ +package ru.trader.analysis.graph; + +import ru.trader.analysis.RouteSpecification; + +import java.util.List; +import java.util.function.Consumer; +import java.util.function.Function; + +public interface CrawlerSpecification { + public RouteSpecification routeSpecification(); + + public Consumer>> onFoundFunc(); + + public Function, Object> getGroupGetter(boolean target); + + public int getGroupCount(); + +} diff --git a/core/src/main/java/ru/trader/analysis/graph/Traversal.java b/core/src/main/java/ru/trader/analysis/graph/Traversal.java index a4dc91a..519b030 100644 --- a/core/src/main/java/ru/trader/analysis/graph/Traversal.java +++ b/core/src/main/java/ru/trader/analysis/graph/Traversal.java @@ -11,6 +11,7 @@ public interface Traversal { void sort(); void setSkipped(boolean skipped); boolean isSkipped(); + boolean containsSkipped(); default boolean isConnect(T target){ Edge edge = getEdge(); diff --git a/core/src/test/java/ru/trader/analysis/VendorsGraphTest.java b/core/src/test/java/ru/trader/analysis/VendorsGraphTest.java index 6fe17a4..0c16ead 100644 --- a/core/src/test/java/ru/trader/analysis/VendorsGraphTest.java +++ b/core/src/test/java/ru/trader/analysis/VendorsGraphTest.java @@ -228,7 +228,9 @@ public class VendorsGraphTest extends Assert { vGraph.build(cabreraDock, fWorld.getMarkets(true).collect(Collectors.toList())); LOG.info("Search"); SimpleCollector paths = new SimpleCollector<>(); - Crawler crawler = vGraph.crawler(new CrawlerSpecificationByProfit(new LoopRouteSpecification<>(true), paths::add, true), new AnalysisCallBack()); + CrawlerSpecificationByProfit specification = new CrawlerSpecificationByProfit(new LoopRouteSpecification<>(true), paths::add, true); + specification.setGroupCount(60); + Crawler crawler = vGraph.crawler(specification, new AnalysisCallBack()); crawler.findMin(cabreraDock, 100); assertEquals(60, paths.get().size()); Collection vendors = new ArrayList<>(60);