Voomra/trader
Archived
0
Fork 0
Code Activity

increase import speed

Browse Source
This commit is contained in:
Mo
2016-04-26 18:04:46 +03:00
parent f7cadbd9e7
commit c20f229c57
6 changed files with 105 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
core/src/main/java/ru/trader/core/AbstractMarket.java
View File

@@ -7,6 +7,7 @@ public abstract class AbstractMarket implements Market {
private final static Logger LOG = LoggerFactory.getLogger(AbstractMarket.class);
private boolean change;
private boolean batch;
protected abstract Place createPlace(String name, double x, double y, double z);
protected abstract Group createGroup(String name, GROUP_TYPE type);
@@ -17,6 +18,25 @@ public abstract class AbstractMarket implements Market {
protected abstract void removePlace(Place place);
protected abstract void addItem(Item item);
protected abstract void removeItem(Item item);
protected void executeBatch(){}
@Override
public void startBatch() {
LOG.debug("Start batch");
batch = true;
}
@Override
public final void doBatch() {
LOG.debug("End batch, updated");
executeBatch();
batch = false;
}
@Override
public boolean isBatch() {
return batch;
}
@Override
public final void add(Place place){

13
core/src/main/java/ru/trader/core/Market.java
View File

@@ -1,10 +1,17 @@
package ru.trader.core;
import java.util.*;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.Optional;
import java.util.stream.Collectors;
public interface Market {
void startBatch();
void doBatch();
boolean isBatch();
void add(Place place);
Place addPlace(String name, double x, double y, double z);
void remove(Place place);
@@ -109,6 +116,7 @@ public interface Market {
}
default void add(Market market){
startBatch();
// add groups
Collection<Group> groups = market.getGroups();
HashMap<Group, Group> mapGroups = new HashMap<>(groups.size(), 0.9f);
@@ -182,6 +190,7 @@ public interface Market {
}
}
}
doBatch();
}
default void clear(){
@@ -210,6 +219,7 @@ public interface Market {
default void clear(boolean offers, boolean vendors, boolean places, boolean items, boolean groups){
startBatch();
if (places){
Collection<Place> p = new ArrayList<>(get());
for (Place place : p) {
@@ -238,6 +248,7 @@ public interface Market {
remove(group);
}
}
doBatch();
}
}

2
core/src/main/java/ru/trader/store/simple/MarketDocHandler.java
View File

@@ -59,6 +59,7 @@ public class MarketDocHandler extends DefaultHandler {
@Override
public void startDocument() throws SAXException {
world = new SimpleMarket();
world.startBatch();
}
@Override
@@ -95,6 +96,7 @@ public class MarketDocHandler extends DefaultHandler {
@Override
public void endDocument() throws SAXException {
world.doBatch();
world.commit();
}

16
core/src/main/java/ru/trader/store/simple/SimpleItemStat.java
View File

@@ -4,8 +4,12 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import ru.trader.core.*;
import java.util.*;
import java.util.Collection;
import java.util.Collections;
import java.util.DoubleSummaryStatistics;
import java.util.NavigableSet;
import java.util.concurrent.ConcurrentSkipListSet;
import java.util.stream.Collectors;
public class SimpleItemStat extends AbstractItemStat {
private final static Logger LOG = LoggerFactory.getLogger(SimpleItemStat.class);
@@ -36,6 +40,16 @@ public class SimpleItemStat extends AbstractItemStat {
}
}
synchronized void putAll(Collection<Offer> offers){
LOG.trace("Put offers to item stat {}", this);
if (this.offers.addAll(offers)){
DoubleSummaryStatistics stat = this.offers.stream().collect(Collectors.summarizingDouble(Offer::getPrice));
sum = stat.getSum();
avg = stat.getAverage();
LOG.trace("After this = {}", this);
}
}
synchronized void remove(Offer offer){
LOG.trace("Remove offer {} from item stat {}", offer, this);
assert offer.hasType(type) && offer.hasItem(item);

49
core/src/main/java/ru/trader/store/simple/SimpleMarket.java
View File

@@ -5,6 +5,7 @@ import org.slf4j.LoggerFactory;
import ru.trader.core.*;
import java.util.*;
import java.util.stream.Stream;
public class SimpleMarket extends AbstractMarket {
private final static Logger LOG = LoggerFactory.getLogger(SimpleMarket.class);
@@ -147,6 +148,7 @@ public class SimpleMarket extends AbstractMarket {
@Override
protected void onAdd(Vendor vendor) {
if (isBatch()) return;
LOG.trace("Cached on add vendor {}", vendor);
Collection<Offer> offers = vendor.getAllSellOffers();
for (Offer offer : offers) {
@@ -160,6 +162,7 @@ public class SimpleMarket extends AbstractMarket {
@Override
protected void onRemove(Vendor vendor) {
if (isBatch()) return;
LOG.trace("Remove cache of vendor {}", vendor);
Collection<Offer> offers = vendor.getAllSellOffers();
for (Offer offer : offers) {
@@ -173,12 +176,14 @@ public class SimpleMarket extends AbstractMarket {
@Override
protected void onAdd(Offer offer) {
if (isBatch()) return;
LOG.trace("Cached on add offer {}", offer);
put(getItemCache(offer.getType()), offer);
}
@Override
protected void onRemove(Offer offer) {
if (isBatch()) return;
LOG.trace("Remove cache of offer {}", offer);
remove(getItemCache(offer.getType()), offer);
}
@@ -191,4 +196,48 @@ public class SimpleMarket extends AbstractMarket {
systems.add(place);
}
@Override
public void startBatch() {
super.startBatch();
sellItems.clear();
buyItems.clear();
}
@Override
protected void executeBatch() {
recreateCaches();
super.executeBatch();
}
private void recreateCaches(){
Map<Item, Collection<Offer>> sellOffers = new HashMap<>();
Map<Item, Collection<Offer>> buyOffers = new HashMap<>();
systems.stream().flatMap(s -> s.get().stream())
.flatMap(v -> Stream.concat(v.getAllSellOffers().stream(), v.getAllBuyOffers().stream()))
.forEach(o -> {
Map<Item, Collection<Offer>> m = o.getType() == OFFER_TYPE.SELL ? sellOffers : buyOffers;
Collection<Offer> offers = m.get(o.getItem());
if (offers == null){
offers = new ArrayList<>(1000);
m.put(o.getItem(), offers);
}
offers.add(o);
});
for (Iterator<Map.Entry<Item, Collection<Offer>>> iterator = sellOffers.entrySet().iterator(); iterator.hasNext(); ) {
Map.Entry<Item, Collection<Offer>> entry = iterator.next();
SimpleItemStat stat = new SimpleItemStat(entry.getKey(), OFFER_TYPE.SELL);
stat.putAll(entry.getValue());
sellItems.put(stat.getItem(), stat);
iterator.remove();
}
for (Iterator<Map.Entry<Item, Collection<Offer>>> iterator = buyOffers.entrySet().iterator(); iterator.hasNext(); ) {
Map.Entry<Item, Collection<Offer>> entry = iterator.next();
SimpleItemStat stat = new SimpleItemStat(entry.getKey(), OFFER_TYPE.BUY);
stat.putAll(entry.getValue());
buyItems.put(stat.getItem(), stat);
iterator.remove();
}
}
}

11
utils/src/main/java/ru/trader/maddavo/Parser.java
View File

@@ -15,22 +15,23 @@ public class Parser {
private boolean canceled;
public void parseSystems(File file, Market market) throws IOException {
parseFile(file, new SystemHandler(market), 1);
parseFile(file, new SystemHandler(market), 1, market);
}
public void parseStations(File file, Market market) throws IOException {
parseFile(file, new StationHandler(market), 1);
parseFile(file, new StationHandler(market), 1, market);
}
public void parsePrices(File file, Market market) throws IOException {
parseFile(file, new OffersHandler(market, true), 0);
parseFile(file, new OffersHandler(market, true), 0, market);
}
private void parseFile(File file, ParseHandler handler, int skip) throws IOException {
private void parseFile(File file, ParseHandler handler, int skip, Market market) throws IOException {
canceled = false;
market.startBatch();
try (BufferedReader reader = new BufferedReader(new FileReader(file))) {
int row = 0;
String line;
@@ -40,6 +41,8 @@ public class Parser {
if (row <= skip) continue;
handler.parse(line);
}
} finally {
market.doBatch();
}
}