From 22e37b1342f637892cf7a23062b3082a0d977258 Mon Sep 17 00:00:00 2001 From: iMoHax Date: Fri, 26 Jun 2015 15:47:01 +0300 Subject: [PATCH] Improve Crawler implementation --- .../java/ru/trader/analysis/VendorsGraph.java | 97 ++-- .../ru/trader/analysis/graph/CCrawler.java | 67 +-- .../trader/analysis/graph/CostTraversal.java | 6 + .../ru/trader/analysis/graph/Crawler.java | 433 ++++++++++++++---- .../ru/trader/analysis/graph/Traversal.java | 47 ++ .../ru/trader/analysis/RouteSearcherTest.java | 55 +-- .../ru/trader/analysis/graph/CrawlerTest.java | 2 +- 7 files changed, 507 insertions(+), 200 deletions(-) create mode 100644 core/src/main/java/ru/trader/analysis/graph/CostTraversal.java create mode 100644 core/src/main/java/ru/trader/analysis/graph/Traversal.java diff --git a/core/src/main/java/ru/trader/analysis/VendorsGraph.java b/core/src/main/java/ru/trader/analysis/VendorsGraph.java index 5eb2eca..6237fd4 100644 --- a/core/src/main/java/ru/trader/analysis/VendorsGraph.java +++ b/core/src/main/java/ru/trader/analysis/VendorsGraph.java @@ -7,8 +7,10 @@ import ru.trader.core.Order; import ru.trader.core.TransitVendor; import ru.trader.core.Vendor; +import java.util.ArrayList; import java.util.Collection; import java.util.List; +import java.util.Optional; import java.util.function.Consumer; @@ -54,17 +56,15 @@ public class VendorsGraph extends ConnectibleGraph { @Override protected ConnectibleEdge createEdge(Vertex target) { - return new VendorsEdge(vertex, target, refill, fuelCost, false); + return new VendorsEdge(vertex, target, refill, fuelCost); } } public class VendorsEdge extends ConnectibleEdge { private List orders; - private boolean isTarget; - protected VendorsEdge(Vertex source, Vertex target, boolean refill, double fuel, boolean isTarget) { + protected VendorsEdge(Vertex source, Vertex target, boolean refill, double fuel) { super(source, target, refill, fuel); - this.isTarget = isTarget; } protected void setOrders(List orders){ @@ -87,7 +87,7 @@ public class VendorsGraph extends ConnectibleGraph { @Override protected double computeWeight() { int jumps = source.getEntry().getPlace().equals(target.getEntry().getPlace())? 0 : 1; - int lands = refill || !orders.isEmpty() || isTarget ? 1 : 0; + int lands = !(target.getEntry() instanceof TransitVendor) ? 1 : 0; boolean transit = lands == 0 && source.getEntry() instanceof TransitVendor || target.getEntry() instanceof TransitVendor; double profit = getProfit(); double score = transit ? scorer.getTransitScore(fuel) : @@ -105,50 +105,79 @@ public class VendorsGraph extends ConnectibleGraph { } @Override - protected CostTraversalEntry start(Vertex vertex) { - double balance = scorer.getProfile().getBalance(); - return new VendorsTraversalEntry((CCostTraversalEntry) super.start(vertex), balance); + protected VendorsTraversalEntry start(Vertex vertex) { + double balance = getProfile().getBalance(); + return new VendorsTraversalEntry(super.start(vertex), balance); } @Override - protected CostTraversalEntry travers(final CostTraversalEntry entry, final List> head, final Edge edge, final Vendor target) { - VendorsTraversalEntry ve = (VendorsTraversalEntry)entry; - double balance = ve.balance; - Vendor buyer = edge.getTarget().getEntry(); - List orders = ((VendorsEdge) edge).getOrders(); - if (edge.getSource().getEntry() instanceof TransitVendor && - !(buyer instanceof TransitVendor)){ - LOG.trace("{} is transit, search seller", edge.getSource().getEntry()); - for (int i = head.size() - 1; i >= 0; i--) { - Vendor seller = head.get(i).getSource().getEntry(); - if (!(seller instanceof TransitVendor)){ - orders = MarketUtils.getOrders(seller, buyer); - break; - } - } - } - orders = MarketUtils.getStack(orders, balance, scorer.getProfile().getShip().getCargo()); - - CCostTraversalEntry ce = (CCostTraversalEntry) super.travers(entry, head, edge, target); - ConnectibleEdge cedge = (ConnectibleEdge) ce.getEdge(); - VendorsEdge addingEdge = new VendorsEdge(cedge.getSource(), cedge.getTarget(), cedge.isRefill(), cedge.getFuel(), target.equals(buyer)); - addingEdge.setOrders(orders); - return new VendorsTraversalEntry(head, addingEdge, entry.getWeight(), ce.getFuel(), balance+addingEdge.getProfit()); + protected VendorsTraversalEntry travers(final CostTraversalEntry entry, final Edge edge, final Vendor target) { + VendorsEdge vEdge = (VendorsEdge) edge; + CCostTraversalEntry ce = super.travers(entry, edge, target); + return new VendorsTraversalEntry((VendorsTraversalEntry) entry, edge, ce.getFuel(), ((VendorsTraversalEntry)entry).balance + vEdge.getProfit()); } protected class VendorsTraversalEntry extends CCostTraversalEntry { private final double balance; protected VendorsTraversalEntry(CCostTraversalEntry entry, double balance) { - super(entry.getHead(), entry.getVertex(), entry.getFuel()); + super(entry.getTarget(), entry.getFuel()); this.balance = balance; } - protected VendorsTraversalEntry(List> head, Edge edge, double cost, double fuel, double balance) { - super(head, edge, cost, fuel); + protected VendorsTraversalEntry(VendorsTraversalEntry head, Edge edge, double fuel, double balance) { + super(head, edge, fuel); this.balance = balance; } + @Override + protected boolean check(Edge e) { + boolean good = super.check(e); + // remove transit cicles + if (good && e.getSource().getEntry() instanceof TransitVendor && !(e.getTarget().getEntry() instanceof TransitVendor)){ + Optional seller = getSeller(); + good = seller.isPresent() && !e.getTarget().isEntry(seller.get()); + } + return good; + } + + @Override + protected VendorsEdge wrap(Edge edge) { + ConnectibleEdge cEdge = super.wrap(edge); + Vendor buyer = edge.getTarget().getEntry(); + List orders = new ArrayList<>(); + orders.addAll(((VendorsEdge) edge).getOrders()); + if (edge.getSource().getEntry() instanceof TransitVendor && !(buyer instanceof TransitVendor)){ + LOG.trace("{} is transit, search seller", edge.getSource().getEntry()); + Optional seller = getSeller(); + if (seller.isPresent()){ + orders = MarketUtils.getOrders(seller.get(), buyer); + } + } + orders = MarketUtils.getStack(orders, balance, getShip().getCargo()); + VendorsEdge res = new VendorsEdge(edge.getSource(), edge.getTarget(), cEdge.isRefill(), cEdge.getFuel()); + res.setOrders(orders); + return res; + } + + private Optional getSeller(){ + Vendor res = null; + Edge e = getEdge(); + Vendor seller = e.getSource().getEntry(); + if (!(seller instanceof TransitVendor)){ + res = seller; + } else { + for (int i = head.size() - 1; i >= 0; i--) { + e = head.getEdge(); + seller = e.getSource().getEntry(); + if (!(seller instanceof TransitVendor)){ + res = seller; + break; + } + } + } + return Optional.ofNullable(res); + } } diff --git a/core/src/main/java/ru/trader/analysis/graph/CCrawler.java b/core/src/main/java/ru/trader/analysis/graph/CCrawler.java index 42cfa9c..f8b5a97 100644 --- a/core/src/main/java/ru/trader/analysis/graph/CCrawler.java +++ b/core/src/main/java/ru/trader/analysis/graph/CCrawler.java @@ -6,10 +6,10 @@ import ru.trader.core.Profile; import ru.trader.core.Ship; import ru.trader.graph.Connectable; -import java.util.ArrayList; -import java.util.Iterator; +import java.util.Collection; import java.util.List; import java.util.function.Consumer; +import java.util.stream.Collectors; public class CCrawler> extends Crawler { private final static Logger LOG = LoggerFactory.getLogger(CCrawler.class); @@ -18,47 +18,45 @@ public class CCrawler> extends Crawler { super(graph, onFoundFunc); } - private Ship getShip(){ + protected Ship getShip(){ return ((ConnectibleGraph)graph).getProfile().getShip(); } - private Profile getProfile(){ + protected Profile getProfile(){ return ((ConnectibleGraph)graph).getProfile(); } @Override - protected CostTraversalEntry start(Vertex vertex) { + protected CCostTraversalEntry start(Vertex vertex) { double fuel = getShip().getTank(); - return new CCostTraversalEntry(new ArrayList<>(), vertex, fuel); + return new CCostTraversalEntry(vertex, fuel); } @Override - protected CostTraversalEntry travers(CostTraversalEntry entry, List> head, Edge edge, T target) { - T source = entry.vertex.getEntry(); - double distance = source.getDistance(edge.target.getEntry()); - double fuelCost = getShip().getFuelCost(((CCostTraversalEntry)entry).fuel, distance); - double nextLimit = getProfile().withRefill() ? ((CCostTraversalEntry)entry).fuel - fuelCost : getShip().getTank(); - boolean refill = nextLimit < 0 && source.canRefill(); - if (refill) { + protected CCostTraversalEntry travers(CostTraversalEntry entry, Edge edge, T target) { + @SuppressWarnings("unchecked") + CCostTraversalEntry cEntry = (CCostTraversalEntry)entry; + ConnectibleEdge cEdge = (ConnectibleEdge) edge; + double nextLimit; + if (cEdge.isRefill()) { LOG.trace("Refill"); - refill = true; - fuelCost = getShip().getFuelCost(distance); - nextLimit = getShip().getTank() - fuelCost; + nextLimit = getShip().getTank() - cEdge.getFuel(); + } else { + nextLimit = getProfile().withRefill() ? cEntry.getFuel() - cEdge.getFuel() : getShip().getTank(); } - edge = new ConnectibleEdge<>(edge.getSource(), edge.getTarget(), refill, fuelCost); - return new CCostTraversalEntry(head, edge, entry.getWeight(), nextLimit); + return new CCostTraversalEntry(cEntry, edge, nextLimit); } protected class CCostTraversalEntry extends CostTraversalEntry { private final double fuel; - protected CCostTraversalEntry(List> head, Vertex vertex, double fuel) { - super(head, vertex); + protected CCostTraversalEntry(Vertex vertex, double fuel) { + super(vertex); this.fuel = fuel; } - protected CCostTraversalEntry(List> head, Edge edge, double cost, double fuel) { - super(head, edge, cost); + protected CCostTraversalEntry(CCostTraversalEntry head, Edge edge, double fuel) { + super(head, edge); this.fuel = fuel; } @@ -67,11 +65,26 @@ public class CCrawler> extends Crawler { } @Override - protected Iterator> getIteratorInstance() { - return vertex.getEdges().stream().filter(e -> { - ConnectibleEdge edge = (ConnectibleEdge) e; - return edge.getFuel() <= fuel || e.getSource().getEntry().canRefill(); - }).iterator(); + public List> collect(Collection> src) { + return src.stream().filter(this::check).map(this::wrap).collect(Collectors.toList()); + } + + protected boolean check(Edge e){ + ConnectibleEdge edge = (ConnectibleEdge) e; + return edge.getFuel() <= fuel || edge.getSource().getEntry().canRefill(); + } + + protected ConnectibleEdge wrap(Edge edge){ + T source = edge.source.getEntry(); + double distance = source.getDistance(edge.target.getEntry()); + double fuelCost = getShip().getFuelCost(fuel, distance); + double nextLimit = getProfile().withRefill() ? fuel - fuelCost : getShip().getTank(); + boolean refill = nextLimit < 0 && source.canRefill(); + if (refill) { + refill = true; + fuelCost = getShip().getFuelCost(distance); + } + return new ConnectibleEdge<>(edge.getSource(), edge.getTarget(), refill, fuelCost); } } } diff --git a/core/src/main/java/ru/trader/analysis/graph/CostTraversal.java b/core/src/main/java/ru/trader/analysis/graph/CostTraversal.java new file mode 100644 index 0000000..c268ebb --- /dev/null +++ b/core/src/main/java/ru/trader/analysis/graph/CostTraversal.java @@ -0,0 +1,6 @@ +package ru.trader.analysis.graph; + +public interface CostTraversal extends Traversal { + double getWeight(); + +} diff --git a/core/src/main/java/ru/trader/analysis/graph/Crawler.java b/core/src/main/java/ru/trader/analysis/graph/Crawler.java index 6cff590..792658c 100644 --- a/core/src/main/java/ru/trader/analysis/graph/Crawler.java +++ b/core/src/main/java/ru/trader/analysis/graph/Crawler.java @@ -5,10 +5,15 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.*; +import java.util.concurrent.ForkJoinPool; +import java.util.concurrent.RecursiveAction; import java.util.function.Consumer; public class Crawler { private final static Logger LOG = LoggerFactory.getLogger(Crawler.class); + private final static ForkJoinPool POOL = new ForkJoinPool(); + private final static int THRESHOLD = 12 * (Runtime.getRuntime().availableProcessors() > 1 ? Runtime.getRuntime().availableProcessors() - 1 : 1); + private final static int SPLIT_SIZE = 3; protected final Graph graph; private final Consumer>> onFoundFunc; @@ -20,9 +25,9 @@ public class Crawler { this.onFoundFunc = onFoundFunc; } - protected List> getCopyList(List> head, Edge tail){ - List> res = new ArrayList<>(20); - res.addAll(head); + protected List> getCopyList(Traversal head, Edge tail){ + List> res = new ArrayList<>(head.size()+ 1); + res.addAll(head.toEdges()); res.add(tail); return res; } @@ -36,15 +41,14 @@ public class Crawler { } public void findFast(T target, int count){ - Vertex t = graph.getVertex(target); + Optional> t = graph.getVertex(target); int found = 0; - if (t != null) { + if (t.isPresent()) { if (count > 1) { - Vertex s = graph.getRoot(); - s.sortEdges(); - found = bfs(start(s), target, graph.getMinLevel(), count); + int maxDeep = maxSize - (graph.getRoot().isEntry(target) ? graph.getMinJumps() * 2 : graph.getMinJumps()); + found = bfs(start(graph.getRoot()), target, maxDeep, count); } else { - found = dfs(start(graph.getRoot()), target, t.getLevel() + 1, count); + found = dfs(start(graph.getRoot()), target, t.get().getLevel() + 1, count); } } LOG.debug("Found {} paths", found); @@ -55,31 +59,36 @@ public class Crawler { } public void findMin(T target, int count){ - Vertex t = graph.getVertex(target); + Optional> t = graph.getVertex(target); int found = 0; - if (t != null) { - found = ucs(start(graph.getRoot()), target, graph.getMinLevel(), count); + if (t.isPresent()) { + int maxDeep = graph.getRoot().isEntry(target) ? maxSize - graph.getMinJumps() * 2 : graph.getMinLevel(); + if (maxDeep < 0) maxDeep = 0; + if (maxSize - maxDeep <= SPLIT_SIZE){ + found = ucs(start(graph.getRoot()), target, maxDeep, count); + } else { + found = ucs2(start(graph.getRoot()), target, maxDeep, count); + } } LOG.debug("Found {} paths", found); } protected CostTraversalEntry start(Vertex vertex){ - return new CostTraversalEntry(new ArrayList<>(), vertex); + return new CostTraversalEntry(vertex); } - protected CostTraversalEntry travers(CostTraversalEntry entry, List> head, Edge edge, T target){ - return new CostTraversalEntry(head, edge, entry.getWeight()); + protected CostTraversalEntry travers(CostTraversalEntry entry, Edge edge, T target){ + return new CostTraversalEntry(entry, edge); } private int dfs(CostTraversalEntry entry, T target, int deep, int count) { int found = 0; - List> head = entry.head; Vertex source = entry.vertex; - LOG.trace("DFS from {} to {}, deep {}, count {}, head {}", source, target, deep, count, head); + LOG.trace("DFS from {} to {}, deep {}, count {}, entry {}", source, target, deep, count, entry); if (deep == source.getLevel()){ for (Edge next : entry.getEdges()) { if (next.isConnect(target)){ - List> res = getCopyList(head, next); + List> res = getCopyList(entry, next); LOG.debug("Last edge find, path {}", res); onFoundFunc.accept(res); found++; @@ -88,11 +97,11 @@ public class Crawler { } } if (found < count){ - if (deep < source.getLevel() && head.size() < maxSize-1) { + if (deep < source.getLevel() && entry.size() < maxSize-1) { LOG.trace("Search around"); for (Edge edge : entry.getEdges()) { if (edge.getTarget().isSingle()) continue; - found += dfs(travers(entry, getCopyList(head, edge), edge, target), target, deep, count-found); + found += dfs(travers(entry, edge, target), target, deep, count-found); if (found >= count) break; } } @@ -104,21 +113,21 @@ public class Crawler { LOG.trace("BFS from {} to {}, deep {}, count {}", root.vertex, target, deep, count); int found = 0; LinkedList queue = new LinkedList<>(); + root.sort(); queue.add(root); while (!queue.isEmpty() && count > found){ CostTraversalEntry entry = queue.poll(); - List> head = entry.head; Vertex source = entry.vertex; - if (head.size() >= maxSize){ + if (entry.size() >= maxSize){ LOG.trace("Is limit deep"); continue; } - LOG.trace("Search from {} to {}, head {}", source, target, head); + LOG.trace("Search from {} to {}, entry {}", source, target, entry); Iterator> iterator = entry.iterator(); while (iterator.hasNext()){ Edge edge = iterator.next(); if (edge.isConnect(target)){ - List> res = getCopyList(head, edge); + List> res = getCopyList(entry, edge); LOG.debug("Last edge find, path {}", res); onFoundFunc.accept(res); found++; @@ -126,8 +135,9 @@ public class Crawler { if (found >= count) break; if (edge.getTarget().isSingle()) continue; if (deep < source.getLevel()) { - edge.getTarget().sortEdges(); - queue.add(travers(entry, getCopyList(head, edge), edge, target)); + CostTraversalEntry nextEntry = travers(entry, edge, target); + nextEntry.sort(); + queue.add(nextEntry); } } } @@ -141,13 +151,11 @@ public class Crawler { queue.add(root); while (!queue.isEmpty() && count > found){ CostTraversalEntry entry = queue.poll(); - LOG.trace("Check path head {}, edge {}, weight {}", entry.head, entry.edge, entry.weight); - List> head = entry.head; - Vertex source = entry.vertex; - Edge edge = entry.edge; + LOG.trace("Check path entry {}, weight {}", entry, entry.weight); + Edge edge = entry.getEdge(); if (edge != null) { if (edge.isConnect(target)) { - List> res = getCopyList(head, edge); + List> res = entry.toEdges(); LOG.debug("Path found {}", res); onFoundFunc.accept(res); found++; @@ -156,128 +164,353 @@ public class Crawler { if (edge.getTarget().isSingle()){ continue; } - head = getCopyList(entry.head, edge); } - if (head.size() >= maxSize){ + if (entry.size() >= maxSize){ LOG.trace("Is limit deep"); continue; } Iterator> iterator = entry.iterator(); - //put only 2 entry for iterate while (iterator.hasNext()){ edge = iterator.next(); - if (deep <= source.getLevel() && !edge.getTarget().isSingle() || edge.isConnect(target)) { + boolean isTarget = edge.isConnect(target); + boolean canDeep = !entry.getTarget().isSingle() && deep < entry.getTarget().getLevel(); + if (canDeep || isTarget){ LOG.trace("Add edge {} to queue", edge); - queue.add(travers(entry, head, edge, target)); + queue.add(travers(entry, edge, target)); } } } return found; } - //last edge don't compare private int ucs2(CostTraversalEntry root, T target, int deep, int count) { LOG.trace("UCS2 from {} to {}, deep {}, count {}", root.vertex, target, deep, count); int found = 0; - PriorityQueue queue = new PriorityQueue<>(); - queue.add(root); + double limit = Double.MAX_VALUE; + PriorityQueue limitQueue = new PriorityQueue<>(); + PriorityQueue queue = new PriorityQueue<>(); + root.sort(); + queue.add(new CTEntrySupport(root)); while (!queue.isEmpty() && count > found){ - CostTraversalEntry entry = queue.peek(); - List> head = entry.edge != null ? getCopyList(entry.head, entry.edge) : entry.head; - Vertex source = entry.vertex; - Iterator> iterator = entry.iterator(); - LOG.trace("Check path head {}, weight {}", head, entry.weight); - int i = 0; - //put only 2 entry for iterate - while (iterator.hasNext() && i < 2){ - Edge edge = iterator.next(); - LOG.trace("Check edge {}", edge); - if (edge.isConnect(target)) { - List> res = getCopyList(head, edge); - LOG.debug("Last edge find, path {}", res); - onFoundFunc.accept(res); - found++; - } + CTEntrySupport curr = queue.poll(); + CostTraversalEntry entry = curr.entry; + LOG.trace("Check path entry {}, weight {}", entry, entry.weight); + if (entry.isConnect(target)) { + List> res = entry.toEdges(); + LOG.trace("Path found {}", res); + onFoundFunc.accept(res); + found++; if (found >= count) break; - if (edge.getTarget().isSingle()) continue; - if (deep < source.getLevel() && head.size() < maxSize-1) { - edge.getTarget().sortEdges(); - queue.add(travers(entry, head, edge, target)); - i++; - } + limit = limitQueue.poll(); } - if (!iterator.hasNext()) queue.poll(); + if (limitQueue.size() + found < count){ + LOG.trace("Continue search, limit {}", limit); + } else { + LOG.trace("Already {} found, extracting", limitQueue.size()); + continue; + } + if (deep >= entry.getTarget().getLevel() || entry.size() >= maxSize){ + LOG.trace("Is limit deep"); + continue; + } + DFS task = new DFS(curr, target, deep, count - found, limit); + POOL.invoke(task); + limitQueue.addAll(task.getLimits()); + queue.addAll(task.getResult()); } return found; } - protected class TraversalEntry { - protected final List> head; - protected final Vertex vertex; - private Iterator> iterator; + private class CTEntrySupport implements Comparable, Iterator>{ + private final CTEntrySupport parent; + private final Iterator> iterator; + private final CostTraversalEntry entry; + private Edge next; - protected TraversalEntry(List> head, Vertex vertex) { - this.head = head; - this.vertex = vertex; + private CTEntrySupport(CostTraversalEntry entry) { + this(null, entry); } - public List> getHead() { - return head; + private CTEntrySupport(CTEntrySupport parent, CostTraversalEntry entry) { + this.parent = parent; + this.iterator = entry.iterator(); + this.entry = entry; + next = iterator.hasNext() ? next = iterator.next() : null; } - public Vertex getVertex() { - return vertex; - } - - public Iterator> iterator(){ - if (iterator == null){ - iterator = getIteratorInstance(); + @Override + public int compareTo(@NotNull CTEntrySupport o) { + int cmp = entry.compareTo(o.entry); + if (cmp != 0) return cmp; + cmp = Integer.compare(entry.size(), o.entry.size()); + if (cmp != 0) return cmp; + CostTraversal cur = entry; + CostTraversal oCur = o.entry; + while (!cur.isRoot()){ + Edge edge = cur.getEdge(); + Edge oEdge = oCur.getEdge(); + cmp = Double.compare(oEdge.weight, edge.weight); + if (cmp != 0) return cmp; + cur = (CostTraversal) cur.getHead().get(); + oCur = (CostTraversal) oCur.getHead().get(); } - return iterator; + return 0; } - protected Iterator> getIteratorInstance(){ - return vertex.getEdges().iterator(); + @Override + public boolean hasNext() { + return next != null; } - protected Iterable> getEdges(){ - return this::iterator; + @Override + public Edge next(){ + Edge res = next; + next = iterator.hasNext() ? next = iterator.next() : null; + return res; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("{"); + sb.append("entry=").append(entry); + sb.append(", next=").append(next); + sb.append('}'); + return sb.toString(); } } - protected class CostTraversalEntry extends TraversalEntry implements Comparable{ - private final Edge edge; - private final double cost; - private Double weight; + private class DFS extends RecursiveAction { + private final CTEntrySupport root; + private final int count; + private final int deep; + private final T target; + private final Collection res; + private final Collection limits; + private final ArrayList subTasks; + private final boolean isSubTask; + private double limit; - protected CostTraversalEntry(List> head, Vertex vertex) { - super(head, vertex); - this.edge = null; - this.cost = 0; + private DFS(CTEntrySupport root, T target, int deep, int count, double limit) { + this(root, target, deep, count, limit, false); } - protected CostTraversalEntry(List> head, Edge edge, double cost) { - super(head, edge.getTarget()); - this.edge = edge; - this.cost = cost; + private DFS(CTEntrySupport root, T target, int deep, int count, double limit, boolean subtask) { + this.root = root; + this.target = target; + this.deep = deep; + this.count = count; + this.limit = limit; + res = new ArrayList<>(count); + limits = new ArrayList<>(count); + subTasks = new ArrayList<>(THRESHOLD); + isSubTask = subtask; } - public double getWeight(){ - if (weight == null){ - weight = cost + (edge !=null ? edge.getWeight() : 0); + private boolean isRoot(CTEntrySupport entry){ + return entry.parent == null || isSubTask && entry == root; + } + + private Collection getLimits() { + if (!isDone()) + throw new IllegalStateException(); + return limits; + } + + private Collection getResult() { + if (!isDone()) + throw new IllegalStateException(); + return res; + } + + private void search(){ + CTEntrySupport curr = root; + LOG.trace("Start {}", root); + while (curr.hasNext()){ + Edge edge = curr.next(); + CostTraversalEntry entry = curr.entry; + LOG.trace("Check edge {}, entry {}, weight {}", edge, entry, entry.weight); + boolean isTarget = edge.isConnect(target); + boolean canDeep = !entry.getTarget().isSingle() && deep < entry.getTarget().getLevel() && entry.size() < maxSize-1; + if (canDeep || isTarget){ + CostTraversalEntry nextEntry = travers(entry, edge, target); + nextEntry.sort(); + curr = new CTEntrySupport(curr, nextEntry); + if (isTarget){ + LOG.trace("Found, add entry {} to queue", nextEntry); + res.add(curr); + limits.add(limit); + limit = nextEntry.getWeight(); + curr = curr.parent; + } else { + if (nextEntry.getWeight() >= limit && limits.size() > 0){ + if (limits.size() < count){ + LOG.trace("Not found, limit {}, add entry {} to queue", limit, nextEntry); + res.add(curr); + } else { + LOG.trace("Not found, limit {}, don't add entry {} to queue", limit, nextEntry); + } + if (!isRoot(curr.parent)){ + curr = curr.parent.parent; + } else { + break; + } + } else { + if (!isRoot(curr) && maxSize-nextEntry.size() < SPLIT_SIZE){ + if (addSubTask(curr)) + curr = curr.parent; + } + } + } + } else { + LOG.trace("Is limit deep"); + } + while (!curr.hasNext() && !isRoot(curr)){ + LOG.trace("Level complete, return to prev level"); + curr = curr.parent; + } } - return weight; + LOG.trace("Done {}", root); + joinSubTasks(); + } + + private boolean addSubTask(CTEntrySupport entry){ + if (subTasks.size() < THRESHOLD){ + DFS subTask = new DFS(entry, target, deep, count, limit, true); + subTask.fork(); + subTasks.add(subTask); + return true; + } else { + joinSubTasks(); + } + return false; + } + + private void joinSubTasks(){ + for (DFS subTask : subTasks) { + subTask.join(); + fill(subTask); + } + subTasks.clear(); + } + + private void fill(DFS subTask){ + LOG.trace("Sub task is done"); + limits.addAll(subTask.getLimits()); + res.addAll(subTask.getResult()); + limit = Math.min(limit, subTask.limit); + } + + @Override + protected void compute() { + search(); + } + } + + + protected class TraversalEntry implements Traversal { + protected final Traversal head; + protected final Vertex vertex; + private final Edge edge; + private List> edges; + private Integer size; + + protected TraversalEntry(Vertex vertex) { + this.vertex = vertex; + this.head = null; + this.edge = null; + edges = null; + } + + protected TraversalEntry(Traversal head, Edge edge) { + this.head = head; + this.vertex = edge.getTarget(); + this.edge = edge; + edges = null; + } + + @Override + public Vertex getTarget() { + return vertex; + } + + @Override + public Optional> getHead() { + return Optional.ofNullable(head); } public Edge getEdge() { return edge; } + @Override + public final List> getEdges(){ + if (edges == null){ + edges = collect(vertex.getEdges()); + } + return edges; + } + + @Override + public final Iterator> iterator(){ + return getEdges().iterator(); + } + + @Override + public void sort(){ + getEdges().sort(Comparator.>naturalOrder()); + } + + protected List> collect(Collection> src){ + return new ArrayList<>(src); + } + + @Override + public int size() { + if (size == null){ + size = Traversal.super.size(); + } + return size; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("{"); + sb.append("head="); + if (!isRoot()){ + sb.append(getHead().get().toEdges()); + } + sb.append(", edge=").append(edge); + sb.append("}"); + return sb.toString(); + } + } + + protected class CostTraversalEntry extends TraversalEntry implements CostTraversal, Comparable{ + private Double weight; + + protected CostTraversalEntry(Vertex vertex) { + super(vertex); + } + + protected CostTraversalEntry(CostTraversalEntry head, Edge edge) { + super(head, edge); + } + + @Override + public double getWeight(){ + if (weight == null){ + Edge edge = getEdge(); + Optional> head = getHead(); + weight = (head.isPresent() ? ((CostTraversalEntry)head.get()).getWeight() : 0) + (edge != null ? edge.getWeight() : 0); + } + return weight; + } + @Override public int compareTo(@NotNull CostTraversalEntry other) { int cmp = Double.compare(getWeight(), other.getWeight()); if (cmp != 0) return cmp; - return Integer.compare(head.size(), other.head.size()); + return Integer.compare(size(), other.size()); } } } diff --git a/core/src/main/java/ru/trader/analysis/graph/Traversal.java b/core/src/main/java/ru/trader/analysis/graph/Traversal.java new file mode 100644 index 0000000..9512aca --- /dev/null +++ b/core/src/main/java/ru/trader/analysis/graph/Traversal.java @@ -0,0 +1,47 @@ +package ru.trader.analysis.graph; + +import java.util.*; + +public interface Traversal { + Vertex getTarget(); + Optional> getHead(); + Edge getEdge(); + List> getEdges(); + Iterator> iterator(); + void sort(); + + default boolean isConnect(T target){ + Edge edge = getEdge(); + return edge != null && edge.isConnect(target); + } + + default boolean isRoot(){ + return !getHead().isPresent(); + } + + default int size(){ + int s = 0; + Optional> t = this.getHead(); + while (t.isPresent()){ + s++; + t = t.get().getHead(); + } + return s; + } + + @SuppressWarnings("unchecked") + default List> toEdges(){ + int s = size(); + Edge[] res = new Edge[s]; + int i = s - 1; + Traversal entry = this; + while (i >= 0){ + Edge edge = entry.getEdge(); + res[i] = edge; + if (i > 0) + entry = entry.getHead().get(); + i--; + } + return Arrays.asList(res); + } +} diff --git a/core/src/test/java/ru/trader/analysis/RouteSearcherTest.java b/core/src/test/java/ru/trader/analysis/RouteSearcherTest.java index f8d2b8b..eec6170 100644 --- a/core/src/test/java/ru/trader/analysis/RouteSearcherTest.java +++ b/core/src/test/java/ru/trader/analysis/RouteSearcherTest.java @@ -22,6 +22,9 @@ public class RouteSearcherTest extends Assert{ private Place lhs3262; private Place morgor; private Place lhs3006; + private Place bd47; + private Place aulin; + private Place iBootis; @Before public void setUp() throws Exception { @@ -31,6 +34,9 @@ public class RouteSearcherTest extends Assert{ lhs3262 = world.get("LHS 3262"); morgor = world.get("Morgor"); lhs3006 = world.get("LHS 3006"); + bd47 = world.get("BD+47 2112"); + aulin = world.get("Aulin"); + iBootis = world.get("i Bootis"); MarketFilter filter = new MarketFilter(); fWorld = new FilteredMarket(world, filter); @@ -45,6 +51,7 @@ public class RouteSearcherTest extends Assert{ Vendor lhs3262_st = lhs3262.get().iterator().next(); Vendor morgor_st = morgor.get().iterator().next(); Vendor lhs3006_st = lhs3006.get().iterator().next(); + Vendor aulin_st = aulin.get().iterator().next(); Ship ship = new Ship(); ship.setCargo(440); ship.setTank(15); ship.setEngine(5, 'A'); ship.setMass(466); @@ -57,50 +64,22 @@ public class RouteSearcherTest extends Assert{ RouteSearcher searcher = new RouteSearcher(scorer); Route route = new Route(new RouteEntry(ithaca_st, false, 3.3789702637348586d, 0)); - route.add(new RouteEntry(morgor_st, false, 4.137765020523591d, 0)); - route.add(new RouteEntry(lhs3006_st, false, 4.0674474942172765d, 0)); + route.add(new RouteEntry(morgor.asTransit(), false, 4.137765020523591d, 0)); + route.add(new RouteEntry(lhs3006.asTransit(), false, 4.0674474942172765d, 0)); route.add(new RouteEntry(lhs3262_st, true, 4.149937831634785d, 0)); - route.add(new RouteEntry(lhs3006_st, false, 4.1292528548103d, 0)); - route.add(new RouteEntry(morgor_st, false, 3.3050364899848566, 0)); - route.add(new RouteEntry(ithaca_st, false, 3.3483447506734136, 0)); + route.add(new RouteEntry(lhs3006.asTransit(), false, 4.1292528548103d, 0)); + route.add(new RouteEntry(morgor.asTransit(), false, 3.3050364899848566, 0)); + route.add(new RouteEntry(ithaca_st, false, 0, 0)); RouteFiller filler = new RouteFiller(scorer); filler.fill(route); + assertEquals(route.getProfit(), 981200, 0); + assertEquals(route.getLands(), 2); + assertEquals(route.getDistance(), 72.42, 0.01); + List apaths = searcher.getRoutes(ithaca_st, ithaca_st, fWorld.getMarkets(true).collect(Collectors.toList())); Route actual = apaths.stream().findFirst().get(); - //assertTrue("Routes is different",expect.isRoute(actual)); + assertEquals("Routes is different", route, actual); } -/* - @Test - public void testRoutes2() throws Exception { - // Balance: 6000000, cargo: 440, tank: 40, distance: 13.6, jumps: 6 - // Ithaca (Palladium to LHS 3262) -> Morgor -> LHS 3006 -> LHS 3262 (Consumer Technology to Ithaca) -> LHS 3006 -> Morgor -> Ithaca - // Profit: 981200, avg: 490600, distance: 67.5, lands: 2 - Vendor ithaca = market.get().stream().filter((v)->v.getName().equals("Ithaca")).findFirst().get().get().iterator().next(); - Vendor lhs3262 = market.get().stream().filter((v)->v.getName().equals("LHS 3262")).findFirst().get().get().iterator().next(); - - RouteSearcher searcher = new RouteSearcher(13.6, 40); - RouteGraph graph = new RouteGraph(ithaca, market.getVendors(true), 40, 13.6, true, 6); - graph.setCargo(440); - graph.setBalance(6000000); - - List> epaths = graph.getPathsTo(ithaca, 10); - PathRoute expect = epaths.stream().map(p -> (PathRoute) p).findFirst().get(); - - List apaths = searcher.getPaths(ithaca, ithaca, market.getVendors(true), 6, 6000000, 440, 10); - PathRoute actual = apaths.stream().findFirst().get(); - assertTrue("Routes is different",expect.isRoute(actual)); - - graph = new RouteGraph(lhs3262, market.getVendors(true), 40, 13.6, true, 6); - graph.setCargo(440); - graph.setBalance(6000000); - - expect = graph.getPathsTo(lhs3262, 10).stream().map(p -> (PathRoute)p).findFirst().get(); - apaths = searcher.getPaths(lhs3262, lhs3262, market.getVendors(true), 6, 6000000, 440, 10); - actual = apaths.stream().findFirst().get(); - assertEquals("Routes is different",expect.getAvgProfit(), actual.getAvgProfit(), 0.00001); - - } -*/ } diff --git a/core/src/test/java/ru/trader/analysis/graph/CrawlerTest.java b/core/src/test/java/ru/trader/analysis/graph/CrawlerTest.java index 17867b8..d5e1626 100644 --- a/core/src/test/java/ru/trader/analysis/graph/CrawlerTest.java +++ b/core/src/test/java/ru/trader/analysis/graph/CrawlerTest.java @@ -137,7 +137,7 @@ public class CrawlerTest extends Assert { PPath.of(x5, x6, x5, x4), PPath.of(x5, x4, x5, x4), PPath.of(x5, x4, x3, x4), PPath.of(x5, x4, x6, x4), PPath.of(x5, x6, x3, x4), PPath.of(x5, x3, x5, x4), PPath.of(x5, x4, x2, x4), - PPath.of(x5, x3, x2, x4), PPath.of(x5, x3, x6, x4) + PPath.of(x5, x3, x6, x4), PPath.of(x5, x3, x2, x4) ); TestUtil.assertCollectionEquals(paths.getWeights(), 5.0, 15.0, 15.0, 15.0, 15.0, 15.0,