Archived
0

Improve Crawler implementation

This commit is contained in:
iMoHax
2015-06-26 15:47:01 +03:00
parent 940ad274ad
commit 22e37b1342
7 changed files with 507 additions and 200 deletions

View File

@@ -7,8 +7,10 @@ import ru.trader.core.Order;
import ru.trader.core.TransitVendor; import ru.trader.core.TransitVendor;
import ru.trader.core.Vendor; import ru.trader.core.Vendor;
import java.util.ArrayList;
import java.util.Collection; import java.util.Collection;
import java.util.List; import java.util.List;
import java.util.Optional;
import java.util.function.Consumer; import java.util.function.Consumer;
@@ -54,17 +56,15 @@ public class VendorsGraph extends ConnectibleGraph<Vendor> {
@Override @Override
protected ConnectibleEdge<Vendor> createEdge(Vertex<Vendor> target) { protected ConnectibleEdge<Vendor> createEdge(Vertex<Vendor> target) {
return new VendorsEdge(vertex, target, refill, fuelCost, false); return new VendorsEdge(vertex, target, refill, fuelCost);
} }
} }
public class VendorsEdge extends ConnectibleEdge<Vendor> { public class VendorsEdge extends ConnectibleEdge<Vendor> {
private List<Order> orders; private List<Order> orders;
private boolean isTarget;
protected VendorsEdge(Vertex<Vendor> source, Vertex<Vendor> target, boolean refill, double fuel, boolean isTarget) { protected VendorsEdge(Vertex<Vendor> source, Vertex<Vendor> target, boolean refill, double fuel) {
super(source, target, refill, fuel); super(source, target, refill, fuel);
this.isTarget = isTarget;
} }
protected void setOrders(List<Order> orders){ protected void setOrders(List<Order> orders){
@@ -87,7 +87,7 @@ public class VendorsGraph extends ConnectibleGraph<Vendor> {
@Override @Override
protected double computeWeight() { protected double computeWeight() {
int jumps = source.getEntry().getPlace().equals(target.getEntry().getPlace())? 0 : 1; int jumps = source.getEntry().getPlace().equals(target.getEntry().getPlace())? 0 : 1;
int lands = refill || !orders.isEmpty() || isTarget ? 1 : 0; int lands = !(target.getEntry() instanceof TransitVendor) ? 1 : 0;
boolean transit = lands == 0 && source.getEntry() instanceof TransitVendor || target.getEntry() instanceof TransitVendor; boolean transit = lands == 0 && source.getEntry() instanceof TransitVendor || target.getEntry() instanceof TransitVendor;
double profit = getProfit(); double profit = getProfit();
double score = transit ? scorer.getTransitScore(fuel) : double score = transit ? scorer.getTransitScore(fuel) :
@@ -105,50 +105,79 @@ public class VendorsGraph extends ConnectibleGraph<Vendor> {
} }
@Override @Override
protected CostTraversalEntry start(Vertex<Vendor> vertex) { protected VendorsTraversalEntry start(Vertex<Vendor> vertex) {
double balance = scorer.getProfile().getBalance(); double balance = getProfile().getBalance();
return new VendorsTraversalEntry((CCostTraversalEntry) super.start(vertex), balance); return new VendorsTraversalEntry(super.start(vertex), balance);
} }
@Override @Override
protected CostTraversalEntry travers(final CostTraversalEntry entry, final List<Edge<Vendor>> head, final Edge<Vendor> edge, final Vendor target) { protected VendorsTraversalEntry travers(final CostTraversalEntry entry, final Edge<Vendor> edge, final Vendor target) {
VendorsTraversalEntry ve = (VendorsTraversalEntry)entry; VendorsEdge vEdge = (VendorsEdge) edge;
double balance = ve.balance; CCostTraversalEntry ce = super.travers(entry, edge, target);
Vendor buyer = edge.getTarget().getEntry(); return new VendorsTraversalEntry((VendorsTraversalEntry) entry, edge, ce.getFuel(), ((VendorsTraversalEntry)entry).balance + vEdge.getProfit());
List<Order> orders = ((VendorsEdge) edge).getOrders();
if (edge.getSource().getEntry() instanceof TransitVendor &&
!(buyer instanceof TransitVendor)){
LOG.trace("{} is transit, search seller", edge.getSource().getEntry());
for (int i = head.size() - 1; i >= 0; i--) {
Vendor seller = head.get(i).getSource().getEntry();
if (!(seller instanceof TransitVendor)){
orders = MarketUtils.getOrders(seller, buyer);
break;
}
}
}
orders = MarketUtils.getStack(orders, balance, scorer.getProfile().getShip().getCargo());
CCostTraversalEntry ce = (CCostTraversalEntry) super.travers(entry, head, edge, target);
ConnectibleEdge<Vendor> cedge = (ConnectibleEdge<Vendor>) ce.getEdge();
VendorsEdge addingEdge = new VendorsEdge(cedge.getSource(), cedge.getTarget(), cedge.isRefill(), cedge.getFuel(), target.equals(buyer));
addingEdge.setOrders(orders);
return new VendorsTraversalEntry(head, addingEdge, entry.getWeight(), ce.getFuel(), balance+addingEdge.getProfit());
} }
protected class VendorsTraversalEntry extends CCostTraversalEntry { protected class VendorsTraversalEntry extends CCostTraversalEntry {
private final double balance; private final double balance;
protected VendorsTraversalEntry(CCostTraversalEntry entry, double balance) { protected VendorsTraversalEntry(CCostTraversalEntry entry, double balance) {
super(entry.getHead(), entry.getVertex(), entry.getFuel()); super(entry.getTarget(), entry.getFuel());
this.balance = balance; this.balance = balance;
} }
protected VendorsTraversalEntry(List<Edge<Vendor>> head, Edge<Vendor> edge, double cost, double fuel, double balance) { protected VendorsTraversalEntry(VendorsTraversalEntry head, Edge<Vendor> edge, double fuel, double balance) {
super(head, edge, cost, fuel); super(head, edge, fuel);
this.balance = balance; this.balance = balance;
} }
@Override
protected boolean check(Edge<Vendor> e) {
boolean good = super.check(e);
// remove transit cicles
if (good && e.getSource().getEntry() instanceof TransitVendor && !(e.getTarget().getEntry() instanceof TransitVendor)){
Optional<Vendor> seller = getSeller();
good = seller.isPresent() && !e.getTarget().isEntry(seller.get());
}
return good;
}
@Override
protected VendorsEdge wrap(Edge<Vendor> edge) {
ConnectibleEdge<Vendor> cEdge = super.wrap(edge);
Vendor buyer = edge.getTarget().getEntry();
List<Order> orders = new ArrayList<>();
orders.addAll(((VendorsEdge) edge).getOrders());
if (edge.getSource().getEntry() instanceof TransitVendor && !(buyer instanceof TransitVendor)){
LOG.trace("{} is transit, search seller", edge.getSource().getEntry());
Optional<Vendor> seller = getSeller();
if (seller.isPresent()){
orders = MarketUtils.getOrders(seller.get(), buyer);
}
}
orders = MarketUtils.getStack(orders, balance, getShip().getCargo());
VendorsEdge res = new VendorsEdge(edge.getSource(), edge.getTarget(), cEdge.isRefill(), cEdge.getFuel());
res.setOrders(orders);
return res;
}
private Optional<Vendor> getSeller(){
Vendor res = null;
Edge<Vendor> e = getEdge();
Vendor seller = e.getSource().getEntry();
if (!(seller instanceof TransitVendor)){
res = seller;
} else {
for (int i = head.size() - 1; i >= 0; i--) {
e = head.getEdge();
seller = e.getSource().getEntry();
if (!(seller instanceof TransitVendor)){
res = seller;
break;
}
}
}
return Optional.ofNullable(res);
}
} }

View File

@@ -6,10 +6,10 @@ import ru.trader.core.Profile;
import ru.trader.core.Ship; import ru.trader.core.Ship;
import ru.trader.graph.Connectable; import ru.trader.graph.Connectable;
import java.util.ArrayList; import java.util.Collection;
import java.util.Iterator;
import java.util.List; import java.util.List;
import java.util.function.Consumer; import java.util.function.Consumer;
import java.util.stream.Collectors;
public class CCrawler<T extends Connectable<T>> extends Crawler<T> { public class CCrawler<T extends Connectable<T>> extends Crawler<T> {
private final static Logger LOG = LoggerFactory.getLogger(CCrawler.class); private final static Logger LOG = LoggerFactory.getLogger(CCrawler.class);
@@ -18,47 +18,45 @@ public class CCrawler<T extends Connectable<T>> extends Crawler<T> {
super(graph, onFoundFunc); super(graph, onFoundFunc);
} }
private Ship getShip(){ protected Ship getShip(){
return ((ConnectibleGraph<T>)graph).getProfile().getShip(); return ((ConnectibleGraph<T>)graph).getProfile().getShip();
} }
private Profile getProfile(){ protected Profile getProfile(){
return ((ConnectibleGraph<T>)graph).getProfile(); return ((ConnectibleGraph<T>)graph).getProfile();
} }
@Override @Override
protected CostTraversalEntry start(Vertex<T> vertex) { protected CCostTraversalEntry start(Vertex<T> vertex) {
double fuel = getShip().getTank(); double fuel = getShip().getTank();
return new CCostTraversalEntry(new ArrayList<>(), vertex, fuel); return new CCostTraversalEntry(vertex, fuel);
} }
@Override @Override
protected CostTraversalEntry travers(CostTraversalEntry entry, List<Edge<T>> head, Edge<T> edge, T target) { protected CCostTraversalEntry travers(CostTraversalEntry entry, Edge<T> edge, T target) {
T source = entry.vertex.getEntry(); @SuppressWarnings("unchecked")
double distance = source.getDistance(edge.target.getEntry()); CCostTraversalEntry cEntry = (CCostTraversalEntry)entry;
double fuelCost = getShip().getFuelCost(((CCostTraversalEntry)entry).fuel, distance); ConnectibleEdge<T> cEdge = (ConnectibleEdge<T>) edge;
double nextLimit = getProfile().withRefill() ? ((CCostTraversalEntry)entry).fuel - fuelCost : getShip().getTank(); double nextLimit;
boolean refill = nextLimit < 0 && source.canRefill(); if (cEdge.isRefill()) {
if (refill) {
LOG.trace("Refill"); LOG.trace("Refill");
refill = true; nextLimit = getShip().getTank() - cEdge.getFuel();
fuelCost = getShip().getFuelCost(distance); } else {
nextLimit = getShip().getTank() - fuelCost; nextLimit = getProfile().withRefill() ? cEntry.getFuel() - cEdge.getFuel() : getShip().getTank();
} }
edge = new ConnectibleEdge<>(edge.getSource(), edge.getTarget(), refill, fuelCost); return new CCostTraversalEntry(cEntry, edge, nextLimit);
return new CCostTraversalEntry(head, edge, entry.getWeight(), nextLimit);
} }
protected class CCostTraversalEntry extends CostTraversalEntry { protected class CCostTraversalEntry extends CostTraversalEntry {
private final double fuel; private final double fuel;
protected CCostTraversalEntry(List<Edge<T>> head, Vertex<T> vertex, double fuel) { protected CCostTraversalEntry(Vertex<T> vertex, double fuel) {
super(head, vertex); super(vertex);
this.fuel = fuel; this.fuel = fuel;
} }
protected CCostTraversalEntry(List<Edge<T>> head, Edge<T> edge, double cost, double fuel) { protected CCostTraversalEntry(CCostTraversalEntry head, Edge<T> edge, double fuel) {
super(head, edge, cost); super(head, edge);
this.fuel = fuel; this.fuel = fuel;
} }
@@ -67,11 +65,26 @@ public class CCrawler<T extends Connectable<T>> extends Crawler<T> {
} }
@Override @Override
protected Iterator<Edge<T>> getIteratorInstance() { public List<Edge<T>> collect(Collection<Edge<T>> src) {
return vertex.getEdges().stream().filter(e -> { return src.stream().filter(this::check).map(this::wrap).collect(Collectors.toList());
ConnectibleEdge<T> edge = (ConnectibleEdge<T>) e; }
return edge.getFuel() <= fuel || e.getSource().getEntry().canRefill();
}).iterator(); protected boolean check(Edge<T> e){
ConnectibleEdge<T> edge = (ConnectibleEdge<T>) e;
return edge.getFuel() <= fuel || edge.getSource().getEntry().canRefill();
}
protected ConnectibleEdge<T> wrap(Edge<T> edge){
T source = edge.source.getEntry();
double distance = source.getDistance(edge.target.getEntry());
double fuelCost = getShip().getFuelCost(fuel, distance);
double nextLimit = getProfile().withRefill() ? fuel - fuelCost : getShip().getTank();
boolean refill = nextLimit < 0 && source.canRefill();
if (refill) {
refill = true;
fuelCost = getShip().getFuelCost(distance);
}
return new ConnectibleEdge<>(edge.getSource(), edge.getTarget(), refill, fuelCost);
} }
} }
} }

View File

@@ -0,0 +1,6 @@
package ru.trader.analysis.graph;
public interface CostTraversal<T> extends Traversal<T> {
double getWeight();
}

View File

@@ -5,10 +5,15 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import java.util.*; import java.util.*;
import java.util.concurrent.ForkJoinPool;
import java.util.concurrent.RecursiveAction;
import java.util.function.Consumer; import java.util.function.Consumer;
public class Crawler<T> { public class Crawler<T> {
private final static Logger LOG = LoggerFactory.getLogger(Crawler.class); private final static Logger LOG = LoggerFactory.getLogger(Crawler.class);
private final static ForkJoinPool POOL = new ForkJoinPool();
private final static int THRESHOLD = 12 * (Runtime.getRuntime().availableProcessors() > 1 ? Runtime.getRuntime().availableProcessors() - 1 : 1);
private final static int SPLIT_SIZE = 3;
protected final Graph<T> graph; protected final Graph<T> graph;
private final Consumer<List<Edge<T>>> onFoundFunc; private final Consumer<List<Edge<T>>> onFoundFunc;
@@ -20,9 +25,9 @@ public class Crawler<T> {
this.onFoundFunc = onFoundFunc; this.onFoundFunc = onFoundFunc;
} }
protected List<Edge<T>> getCopyList(List<Edge<T>> head, Edge<T> tail){ protected List<Edge<T>> getCopyList(Traversal<T> head, Edge<T> tail){
List<Edge<T>> res = new ArrayList<>(20); List<Edge<T>> res = new ArrayList<>(head.size()+ 1);
res.addAll(head); res.addAll(head.toEdges());
res.add(tail); res.add(tail);
return res; return res;
} }
@@ -36,15 +41,14 @@ public class Crawler<T> {
} }
public void findFast(T target, int count){ public void findFast(T target, int count){
Vertex<T> t = graph.getVertex(target); Optional<Vertex<T>> t = graph.getVertex(target);
int found = 0; int found = 0;
if (t != null) { if (t.isPresent()) {
if (count > 1) { if (count > 1) {
Vertex<T> s = graph.getRoot(); int maxDeep = maxSize - (graph.getRoot().isEntry(target) ? graph.getMinJumps() * 2 : graph.getMinJumps());
s.sortEdges(); found = bfs(start(graph.getRoot()), target, maxDeep, count);
found = bfs(start(s), target, graph.getMinLevel(), count);
} else { } else {
found = dfs(start(graph.getRoot()), target, t.getLevel() + 1, count); found = dfs(start(graph.getRoot()), target, t.get().getLevel() + 1, count);
} }
} }
LOG.debug("Found {} paths", found); LOG.debug("Found {} paths", found);
@@ -55,31 +59,36 @@ public class Crawler<T> {
} }
public void findMin(T target, int count){ public void findMin(T target, int count){
Vertex<T> t = graph.getVertex(target); Optional<Vertex<T>> t = graph.getVertex(target);
int found = 0; int found = 0;
if (t != null) { if (t.isPresent()) {
found = ucs(start(graph.getRoot()), target, graph.getMinLevel(), count); int maxDeep = graph.getRoot().isEntry(target) ? maxSize - graph.getMinJumps() * 2 : graph.getMinLevel();
if (maxDeep < 0) maxDeep = 0;
if (maxSize - maxDeep <= SPLIT_SIZE){
found = ucs(start(graph.getRoot()), target, maxDeep, count);
} else {
found = ucs2(start(graph.getRoot()), target, maxDeep, count);
}
} }
LOG.debug("Found {} paths", found); LOG.debug("Found {} paths", found);
} }
protected CostTraversalEntry start(Vertex<T> vertex){ protected CostTraversalEntry start(Vertex<T> vertex){
return new CostTraversalEntry(new ArrayList<>(), vertex); return new CostTraversalEntry(vertex);
} }
protected CostTraversalEntry travers(CostTraversalEntry entry, List<Edge<T>> head, Edge<T> edge, T target){ protected CostTraversalEntry travers(CostTraversalEntry entry, Edge<T> edge, T target){
return new CostTraversalEntry(head, edge, entry.getWeight()); return new CostTraversalEntry(entry, edge);
} }
private int dfs(CostTraversalEntry entry, T target, int deep, int count) { private int dfs(CostTraversalEntry entry, T target, int deep, int count) {
int found = 0; int found = 0;
List<Edge<T>> head = entry.head;
Vertex<T> source = entry.vertex; Vertex<T> source = entry.vertex;
LOG.trace("DFS from {} to {}, deep {}, count {}, head {}", source, target, deep, count, head); LOG.trace("DFS from {} to {}, deep {}, count {}, entry {}", source, target, deep, count, entry);
if (deep == source.getLevel()){ if (deep == source.getLevel()){
for (Edge<T> next : entry.getEdges()) { for (Edge<T> next : entry.getEdges()) {
if (next.isConnect(target)){ if (next.isConnect(target)){
List<Edge<T>> res = getCopyList(head, next); List<Edge<T>> res = getCopyList(entry, next);
LOG.debug("Last edge find, path {}", res); LOG.debug("Last edge find, path {}", res);
onFoundFunc.accept(res); onFoundFunc.accept(res);
found++; found++;
@@ -88,11 +97,11 @@ public class Crawler<T> {
} }
} }
if (found < count){ if (found < count){
if (deep < source.getLevel() && head.size() < maxSize-1) { if (deep < source.getLevel() && entry.size() < maxSize-1) {
LOG.trace("Search around"); LOG.trace("Search around");
for (Edge<T> edge : entry.getEdges()) { for (Edge<T> edge : entry.getEdges()) {
if (edge.getTarget().isSingle()) continue; if (edge.getTarget().isSingle()) continue;
found += dfs(travers(entry, getCopyList(head, edge), edge, target), target, deep, count-found); found += dfs(travers(entry, edge, target), target, deep, count-found);
if (found >= count) break; if (found >= count) break;
} }
} }
@@ -104,21 +113,21 @@ public class Crawler<T> {
LOG.trace("BFS from {} to {}, deep {}, count {}", root.vertex, target, deep, count); LOG.trace("BFS from {} to {}, deep {}, count {}", root.vertex, target, deep, count);
int found = 0; int found = 0;
LinkedList<CostTraversalEntry> queue = new LinkedList<>(); LinkedList<CostTraversalEntry> queue = new LinkedList<>();
root.sort();
queue.add(root); queue.add(root);
while (!queue.isEmpty() && count > found){ while (!queue.isEmpty() && count > found){
CostTraversalEntry entry = queue.poll(); CostTraversalEntry entry = queue.poll();
List<Edge<T>> head = entry.head;
Vertex<T> source = entry.vertex; Vertex<T> source = entry.vertex;
if (head.size() >= maxSize){ if (entry.size() >= maxSize){
LOG.trace("Is limit deep"); LOG.trace("Is limit deep");
continue; continue;
} }
LOG.trace("Search from {} to {}, head {}", source, target, head); LOG.trace("Search from {} to {}, entry {}", source, target, entry);
Iterator<Edge<T>> iterator = entry.iterator(); Iterator<Edge<T>> iterator = entry.iterator();
while (iterator.hasNext()){ while (iterator.hasNext()){
Edge<T> edge = iterator.next(); Edge<T> edge = iterator.next();
if (edge.isConnect(target)){ if (edge.isConnect(target)){
List<Edge<T>> res = getCopyList(head, edge); List<Edge<T>> res = getCopyList(entry, edge);
LOG.debug("Last edge find, path {}", res); LOG.debug("Last edge find, path {}", res);
onFoundFunc.accept(res); onFoundFunc.accept(res);
found++; found++;
@@ -126,8 +135,9 @@ public class Crawler<T> {
if (found >= count) break; if (found >= count) break;
if (edge.getTarget().isSingle()) continue; if (edge.getTarget().isSingle()) continue;
if (deep < source.getLevel()) { if (deep < source.getLevel()) {
edge.getTarget().sortEdges(); CostTraversalEntry nextEntry = travers(entry, edge, target);
queue.add(travers(entry, getCopyList(head, edge), edge, target)); nextEntry.sort();
queue.add(nextEntry);
} }
} }
} }
@@ -141,13 +151,11 @@ public class Crawler<T> {
queue.add(root); queue.add(root);
while (!queue.isEmpty() && count > found){ while (!queue.isEmpty() && count > found){
CostTraversalEntry entry = queue.poll(); CostTraversalEntry entry = queue.poll();
LOG.trace("Check path head {}, edge {}, weight {}", entry.head, entry.edge, entry.weight); LOG.trace("Check path entry {}, weight {}", entry, entry.weight);
List<Edge<T>> head = entry.head; Edge<T> edge = entry.getEdge();
Vertex<T> source = entry.vertex;
Edge<T> edge = entry.edge;
if (edge != null) { if (edge != null) {
if (edge.isConnect(target)) { if (edge.isConnect(target)) {
List<Edge<T>> res = getCopyList(head, edge); List<Edge<T>> res = entry.toEdges();
LOG.debug("Path found {}", res); LOG.debug("Path found {}", res);
onFoundFunc.accept(res); onFoundFunc.accept(res);
found++; found++;
@@ -156,128 +164,353 @@ public class Crawler<T> {
if (edge.getTarget().isSingle()){ if (edge.getTarget().isSingle()){
continue; continue;
} }
head = getCopyList(entry.head, edge);
} }
if (head.size() >= maxSize){ if (entry.size() >= maxSize){
LOG.trace("Is limit deep"); LOG.trace("Is limit deep");
continue; continue;
} }
Iterator<Edge<T>> iterator = entry.iterator(); Iterator<Edge<T>> iterator = entry.iterator();
//put only 2 entry for iterate
while (iterator.hasNext()){ while (iterator.hasNext()){
edge = iterator.next(); edge = iterator.next();
if (deep <= source.getLevel() && !edge.getTarget().isSingle() || edge.isConnect(target)) { boolean isTarget = edge.isConnect(target);
boolean canDeep = !entry.getTarget().isSingle() && deep < entry.getTarget().getLevel();
if (canDeep || isTarget){
LOG.trace("Add edge {} to queue", edge); LOG.trace("Add edge {} to queue", edge);
queue.add(travers(entry, head, edge, target)); queue.add(travers(entry, edge, target));
} }
} }
} }
return found; return found;
} }
//last edge don't compare
private int ucs2(CostTraversalEntry root, T target, int deep, int count) { private int ucs2(CostTraversalEntry root, T target, int deep, int count) {
LOG.trace("UCS2 from {} to {}, deep {}, count {}", root.vertex, target, deep, count); LOG.trace("UCS2 from {} to {}, deep {}, count {}", root.vertex, target, deep, count);
int found = 0; int found = 0;
PriorityQueue<CostTraversalEntry> queue = new PriorityQueue<>(); double limit = Double.MAX_VALUE;
queue.add(root); PriorityQueue<Double> limitQueue = new PriorityQueue<>();
PriorityQueue<CTEntrySupport> queue = new PriorityQueue<>();
root.sort();
queue.add(new CTEntrySupport(root));
while (!queue.isEmpty() && count > found){ while (!queue.isEmpty() && count > found){
CostTraversalEntry entry = queue.peek(); CTEntrySupport curr = queue.poll();
List<Edge<T>> head = entry.edge != null ? getCopyList(entry.head, entry.edge) : entry.head; CostTraversalEntry entry = curr.entry;
Vertex<T> source = entry.vertex; LOG.trace("Check path entry {}, weight {}", entry, entry.weight);
Iterator<Edge<T>> iterator = entry.iterator(); if (entry.isConnect(target)) {
LOG.trace("Check path head {}, weight {}", head, entry.weight); List<Edge<T>> res = entry.toEdges();
int i = 0; LOG.trace("Path found {}", res);
//put only 2 entry for iterate onFoundFunc.accept(res);
while (iterator.hasNext() && i < 2){ found++;
Edge<T> edge = iterator.next();
LOG.trace("Check edge {}", edge);
if (edge.isConnect(target)) {
List<Edge<T>> res = getCopyList(head, edge);
LOG.debug("Last edge find, path {}", res);
onFoundFunc.accept(res);
found++;
}
if (found >= count) break; if (found >= count) break;
if (edge.getTarget().isSingle()) continue; limit = limitQueue.poll();
if (deep < source.getLevel() && head.size() < maxSize-1) {
edge.getTarget().sortEdges();
queue.add(travers(entry, head, edge, target));
i++;
}
} }
if (!iterator.hasNext()) queue.poll(); if (limitQueue.size() + found < count){
LOG.trace("Continue search, limit {}", limit);
} else {
LOG.trace("Already {} found, extracting", limitQueue.size());
continue;
}
if (deep >= entry.getTarget().getLevel() || entry.size() >= maxSize){
LOG.trace("Is limit deep");
continue;
}
DFS task = new DFS(curr, target, deep, count - found, limit);
POOL.invoke(task);
limitQueue.addAll(task.getLimits());
queue.addAll(task.getResult());
} }
return found; return found;
} }
protected class TraversalEntry { private class CTEntrySupport implements Comparable<CTEntrySupport>, Iterator<Edge<T>>{
protected final List<Edge<T>> head; private final CTEntrySupport parent;
protected final Vertex<T> vertex; private final Iterator<Edge<T>> iterator;
private Iterator<Edge<T>> iterator; private final CostTraversalEntry entry;
private Edge<T> next;
protected TraversalEntry(List<Edge<T>> head, Vertex<T> vertex) { private CTEntrySupport(CostTraversalEntry entry) {
this.head = head; this(null, entry);
this.vertex = vertex;
} }
public List<Edge<T>> getHead() { private CTEntrySupport(CTEntrySupport parent, CostTraversalEntry entry) {
return head; this.parent = parent;
this.iterator = entry.iterator();
this.entry = entry;
next = iterator.hasNext() ? next = iterator.next() : null;
} }
public Vertex<T> getVertex() { @Override
return vertex; public int compareTo(@NotNull CTEntrySupport o) {
} int cmp = entry.compareTo(o.entry);
if (cmp != 0) return cmp;
public Iterator<Edge<T>> iterator(){ cmp = Integer.compare(entry.size(), o.entry.size());
if (iterator == null){ if (cmp != 0) return cmp;
iterator = getIteratorInstance(); CostTraversal<T> cur = entry;
CostTraversal<T> oCur = o.entry;
while (!cur.isRoot()){
Edge<T> edge = cur.getEdge();
Edge<T> oEdge = oCur.getEdge();
cmp = Double.compare(oEdge.weight, edge.weight);
if (cmp != 0) return cmp;
cur = (CostTraversal<T>) cur.getHead().get();
oCur = (CostTraversal<T>) oCur.getHead().get();
} }
return iterator; return 0;
} }
protected Iterator<Edge<T>> getIteratorInstance(){ @Override
return vertex.getEdges().iterator(); public boolean hasNext() {
return next != null;
} }
protected Iterable<Edge<T>> getEdges(){ @Override
return this::iterator; public Edge<T> next(){
Edge<T> res = next;
next = iterator.hasNext() ? next = iterator.next() : null;
return res;
}
@Override
public String toString() {
final StringBuilder sb = new StringBuilder("{");
sb.append("entry=").append(entry);
sb.append(", next=").append(next);
sb.append('}');
return sb.toString();
} }
} }
protected class CostTraversalEntry extends TraversalEntry implements Comparable<CostTraversalEntry>{ private class DFS extends RecursiveAction {
private final Edge<T> edge; private final CTEntrySupport root;
private final double cost; private final int count;
private Double weight; private final int deep;
private final T target;
private final Collection<CTEntrySupport> res;
private final Collection<Double> limits;
private final ArrayList<DFS> subTasks;
private final boolean isSubTask;
private double limit;
protected CostTraversalEntry(List<Edge<T>> head, Vertex<T> vertex) { private DFS(CTEntrySupport root, T target, int deep, int count, double limit) {
super(head, vertex); this(root, target, deep, count, limit, false);
this.edge = null;
this.cost = 0;
} }
protected CostTraversalEntry(List<Edge<T>> head, Edge<T> edge, double cost) { private DFS(CTEntrySupport root, T target, int deep, int count, double limit, boolean subtask) {
super(head, edge.getTarget()); this.root = root;
this.edge = edge; this.target = target;
this.cost = cost; this.deep = deep;
this.count = count;
this.limit = limit;
res = new ArrayList<>(count);
limits = new ArrayList<>(count);
subTasks = new ArrayList<>(THRESHOLD);
isSubTask = subtask;
} }
public double getWeight(){ private boolean isRoot(CTEntrySupport entry){
if (weight == null){ return entry.parent == null || isSubTask && entry == root;
weight = cost + (edge !=null ? edge.getWeight() : 0); }
private Collection<Double> getLimits() {
if (!isDone())
throw new IllegalStateException();
return limits;
}
private Collection<CTEntrySupport> getResult() {
if (!isDone())
throw new IllegalStateException();
return res;
}
private void search(){
CTEntrySupport curr = root;
LOG.trace("Start {}", root);
while (curr.hasNext()){
Edge<T> edge = curr.next();
CostTraversalEntry entry = curr.entry;
LOG.trace("Check edge {}, entry {}, weight {}", edge, entry, entry.weight);
boolean isTarget = edge.isConnect(target);
boolean canDeep = !entry.getTarget().isSingle() && deep < entry.getTarget().getLevel() && entry.size() < maxSize-1;
if (canDeep || isTarget){
CostTraversalEntry nextEntry = travers(entry, edge, target);
nextEntry.sort();
curr = new CTEntrySupport(curr, nextEntry);
if (isTarget){
LOG.trace("Found, add entry {} to queue", nextEntry);
res.add(curr);
limits.add(limit);
limit = nextEntry.getWeight();
curr = curr.parent;
} else {
if (nextEntry.getWeight() >= limit && limits.size() > 0){
if (limits.size() < count){
LOG.trace("Not found, limit {}, add entry {} to queue", limit, nextEntry);
res.add(curr);
} else {
LOG.trace("Not found, limit {}, don't add entry {} to queue", limit, nextEntry);
}
if (!isRoot(curr.parent)){
curr = curr.parent.parent;
} else {
break;
}
} else {
if (!isRoot(curr) && maxSize-nextEntry.size() < SPLIT_SIZE){
if (addSubTask(curr))
curr = curr.parent;
}
}
}
} else {
LOG.trace("Is limit deep");
}
while (!curr.hasNext() && !isRoot(curr)){
LOG.trace("Level complete, return to prev level");
curr = curr.parent;
}
} }
return weight; LOG.trace("Done {}", root);
joinSubTasks();
}
private boolean addSubTask(CTEntrySupport entry){
if (subTasks.size() < THRESHOLD){
DFS subTask = new DFS(entry, target, deep, count, limit, true);
subTask.fork();
subTasks.add(subTask);
return true;
} else {
joinSubTasks();
}
return false;
}
private void joinSubTasks(){
for (DFS subTask : subTasks) {
subTask.join();
fill(subTask);
}
subTasks.clear();
}
private void fill(DFS subTask){
LOG.trace("Sub task is done");
limits.addAll(subTask.getLimits());
res.addAll(subTask.getResult());
limit = Math.min(limit, subTask.limit);
}
@Override
protected void compute() {
search();
}
}
protected class TraversalEntry implements Traversal<T> {
protected final Traversal<T> head;
protected final Vertex<T> vertex;
private final Edge<T> edge;
private List<Edge<T>> edges;
private Integer size;
protected TraversalEntry(Vertex<T> vertex) {
this.vertex = vertex;
this.head = null;
this.edge = null;
edges = null;
}
protected TraversalEntry(Traversal<T> head, Edge<T> edge) {
this.head = head;
this.vertex = edge.getTarget();
this.edge = edge;
edges = null;
}
@Override
public Vertex<T> getTarget() {
return vertex;
}
@Override
public Optional<Traversal<T>> getHead() {
return Optional.ofNullable(head);
} }
public Edge<T> getEdge() { public Edge<T> getEdge() {
return edge; return edge;
} }
@Override
public final List<Edge<T>> getEdges(){
if (edges == null){
edges = collect(vertex.getEdges());
}
return edges;
}
@Override
public final Iterator<Edge<T>> iterator(){
return getEdges().iterator();
}
@Override
public void sort(){
getEdges().sort(Comparator.<Edge<T>>naturalOrder());
}
protected List<Edge<T>> collect(Collection<Edge<T>> src){
return new ArrayList<>(src);
}
@Override
public int size() {
if (size == null){
size = Traversal.super.size();
}
return size;
}
@Override
public String toString() {
final StringBuilder sb = new StringBuilder("{");
sb.append("head=");
if (!isRoot()){
sb.append(getHead().get().toEdges());
}
sb.append(", edge=").append(edge);
sb.append("}");
return sb.toString();
}
}
protected class CostTraversalEntry extends TraversalEntry implements CostTraversal<T>, Comparable<CostTraversalEntry>{
private Double weight;
protected CostTraversalEntry(Vertex<T> vertex) {
super(vertex);
}
protected CostTraversalEntry(CostTraversalEntry head, Edge<T> edge) {
super(head, edge);
}
@Override
public double getWeight(){
if (weight == null){
Edge<T> edge = getEdge();
Optional<Traversal<T>> head = getHead();
weight = (head.isPresent() ? ((CostTraversalEntry)head.get()).getWeight() : 0) + (edge != null ? edge.getWeight() : 0);
}
return weight;
}
@Override @Override
public int compareTo(@NotNull CostTraversalEntry other) { public int compareTo(@NotNull CostTraversalEntry other) {
int cmp = Double.compare(getWeight(), other.getWeight()); int cmp = Double.compare(getWeight(), other.getWeight());
if (cmp != 0) return cmp; if (cmp != 0) return cmp;
return Integer.compare(head.size(), other.head.size()); return Integer.compare(size(), other.size());
} }
} }
} }

View File

@@ -0,0 +1,47 @@
package ru.trader.analysis.graph;
import java.util.*;
public interface Traversal<T> {
Vertex<T> getTarget();
Optional<Traversal<T>> getHead();
Edge<T> getEdge();
List<Edge<T>> getEdges();
Iterator<Edge<T>> iterator();
void sort();
default boolean isConnect(T target){
Edge<T> edge = getEdge();
return edge != null && edge.isConnect(target);
}
default boolean isRoot(){
return !getHead().isPresent();
}
default int size(){
int s = 0;
Optional<Traversal<T>> t = this.getHead();
while (t.isPresent()){
s++;
t = t.get().getHead();
}
return s;
}
@SuppressWarnings("unchecked")
default List<Edge<T>> toEdges(){
int s = size();
Edge<T>[] res = new Edge[s];
int i = s - 1;
Traversal<T> entry = this;
while (i >= 0){
Edge<T> edge = entry.getEdge();
res[i] = edge;
if (i > 0)
entry = entry.getHead().get();
i--;
}
return Arrays.asList(res);
}
}

View File

@@ -22,6 +22,9 @@ public class RouteSearcherTest extends Assert{
private Place lhs3262; private Place lhs3262;
private Place morgor; private Place morgor;
private Place lhs3006; private Place lhs3006;
private Place bd47;
private Place aulin;
private Place iBootis;
@Before @Before
public void setUp() throws Exception { public void setUp() throws Exception {
@@ -31,6 +34,9 @@ public class RouteSearcherTest extends Assert{
lhs3262 = world.get("LHS 3262"); lhs3262 = world.get("LHS 3262");
morgor = world.get("Morgor"); morgor = world.get("Morgor");
lhs3006 = world.get("LHS 3006"); lhs3006 = world.get("LHS 3006");
bd47 = world.get("BD+47 2112");
aulin = world.get("Aulin");
iBootis = world.get("i Bootis");
MarketFilter filter = new MarketFilter(); MarketFilter filter = new MarketFilter();
fWorld = new FilteredMarket(world, filter); fWorld = new FilteredMarket(world, filter);
@@ -45,6 +51,7 @@ public class RouteSearcherTest extends Assert{
Vendor lhs3262_st = lhs3262.get().iterator().next(); Vendor lhs3262_st = lhs3262.get().iterator().next();
Vendor morgor_st = morgor.get().iterator().next(); Vendor morgor_st = morgor.get().iterator().next();
Vendor lhs3006_st = lhs3006.get().iterator().next(); Vendor lhs3006_st = lhs3006.get().iterator().next();
Vendor aulin_st = aulin.get().iterator().next();
Ship ship = new Ship(); Ship ship = new Ship();
ship.setCargo(440); ship.setTank(15); ship.setCargo(440); ship.setTank(15);
ship.setEngine(5, 'A'); ship.setMass(466); ship.setEngine(5, 'A'); ship.setMass(466);
@@ -57,50 +64,22 @@ public class RouteSearcherTest extends Assert{
RouteSearcher searcher = new RouteSearcher(scorer); RouteSearcher searcher = new RouteSearcher(scorer);
Route route = new Route(new RouteEntry(ithaca_st, false, 3.3789702637348586d, 0)); Route route = new Route(new RouteEntry(ithaca_st, false, 3.3789702637348586d, 0));
route.add(new RouteEntry(morgor_st, false, 4.137765020523591d, 0)); route.add(new RouteEntry(morgor.asTransit(), false, 4.137765020523591d, 0));
route.add(new RouteEntry(lhs3006_st, false, 4.0674474942172765d, 0)); route.add(new RouteEntry(lhs3006.asTransit(), false, 4.0674474942172765d, 0));
route.add(new RouteEntry(lhs3262_st, true, 4.149937831634785d, 0)); route.add(new RouteEntry(lhs3262_st, true, 4.149937831634785d, 0));
route.add(new RouteEntry(lhs3006_st, false, 4.1292528548103d, 0)); route.add(new RouteEntry(lhs3006.asTransit(), false, 4.1292528548103d, 0));
route.add(new RouteEntry(morgor_st, false, 3.3050364899848566, 0)); route.add(new RouteEntry(morgor.asTransit(), false, 3.3050364899848566, 0));
route.add(new RouteEntry(ithaca_st, false, 3.3483447506734136, 0)); route.add(new RouteEntry(ithaca_st, false, 0, 0));
RouteFiller filler = new RouteFiller(scorer); RouteFiller filler = new RouteFiller(scorer);
filler.fill(route); filler.fill(route);
assertEquals(route.getProfit(), 981200, 0);
assertEquals(route.getLands(), 2);
assertEquals(route.getDistance(), 72.42, 0.01);
List<Route> apaths = searcher.getRoutes(ithaca_st, ithaca_st, fWorld.getMarkets(true).collect(Collectors.toList())); List<Route> apaths = searcher.getRoutes(ithaca_st, ithaca_st, fWorld.getMarkets(true).collect(Collectors.toList()));
Route actual = apaths.stream().findFirst().get(); Route actual = apaths.stream().findFirst().get();
//assertTrue("Routes is different",expect.isRoute(actual)); assertEquals("Routes is different", route, actual);
} }
/*
@Test
public void testRoutes2() throws Exception {
// Balance: 6000000, cargo: 440, tank: 40, distance: 13.6, jumps: 6
// Ithaca (Palladium to LHS 3262) -> Morgor -> LHS 3006 -> LHS 3262 (Consumer Technology to Ithaca) -> LHS 3006 -> Morgor -> Ithaca
// Profit: 981200, avg: 490600, distance: 67.5, lands: 2
Vendor ithaca = market.get().stream().filter((v)->v.getName().equals("Ithaca")).findFirst().get().get().iterator().next();
Vendor lhs3262 = market.get().stream().filter((v)->v.getName().equals("LHS 3262")).findFirst().get().get().iterator().next();
RouteSearcher searcher = new RouteSearcher(13.6, 40);
RouteGraph graph = new RouteGraph(ithaca, market.getVendors(true), 40, 13.6, true, 6);
graph.setCargo(440);
graph.setBalance(6000000);
List<Path<Vendor>> epaths = graph.getPathsTo(ithaca, 10);
PathRoute expect = epaths.stream().map(p -> (PathRoute) p).findFirst().get();
List<PathRoute> apaths = searcher.getPaths(ithaca, ithaca, market.getVendors(true), 6, 6000000, 440, 10);
PathRoute actual = apaths.stream().findFirst().get();
assertTrue("Routes is different",expect.isRoute(actual));
graph = new RouteGraph(lhs3262, market.getVendors(true), 40, 13.6, true, 6);
graph.setCargo(440);
graph.setBalance(6000000);
expect = graph.getPathsTo(lhs3262, 10).stream().map(p -> (PathRoute)p).findFirst().get();
apaths = searcher.getPaths(lhs3262, lhs3262, market.getVendors(true), 6, 6000000, 440, 10);
actual = apaths.stream().findFirst().get();
assertEquals("Routes is different",expect.getAvgProfit(), actual.getAvgProfit(), 0.00001);
}
*/
} }

View File

@@ -137,7 +137,7 @@ public class CrawlerTest extends Assert {
PPath.of(x5, x6, x5, x4), PPath.of(x5, x4, x5, x4), PPath.of(x5, x4, x3, x4), PPath.of(x5, x6, x5, x4), PPath.of(x5, x4, x5, x4), PPath.of(x5, x4, x3, x4),
PPath.of(x5, x4, x6, x4), PPath.of(x5, x6, x3, x4), PPath.of(x5, x4, x6, x4), PPath.of(x5, x6, x3, x4),
PPath.of(x5, x3, x5, x4), PPath.of(x5, x4, x2, x4), PPath.of(x5, x3, x5, x4), PPath.of(x5, x4, x2, x4),
PPath.of(x5, x3, x2, x4), PPath.of(x5, x3, x6, x4) PPath.of(x5, x3, x6, x4), PPath.of(x5, x3, x2, x4)
); );
TestUtil.assertCollectionEquals(paths.getWeights(), 5.0, 15.0, 15.0, TestUtil.assertCollectionEquals(paths.getWeights(), 5.0, 15.0, 15.0,
15.0, 15.0, 15.0, 15.0, 15.0, 15.0,