From c20f229c572d9b037fbe45ae59d5244c90f43315 Mon Sep 17 00:00:00 2001 From: Mo Date: Tue, 26 Apr 2016 18:04:46 +0300 Subject: [PATCH] increase import speed --- .../java/ru/trader/core/AbstractMarket.java | 20 ++++++++ core/src/main/java/ru/trader/core/Market.java | 13 ++++- .../trader/store/simple/MarketDocHandler.java | 2 + .../trader/store/simple/SimpleItemStat.java | 16 +++++- .../ru/trader/store/simple/SimpleMarket.java | 49 +++++++++++++++++++ .../main/java/ru/trader/maddavo/Parser.java | 11 +++-- 6 files changed, 105 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/ru/trader/core/AbstractMarket.java b/core/src/main/java/ru/trader/core/AbstractMarket.java index 0ce25db..9217c78 100644 --- a/core/src/main/java/ru/trader/core/AbstractMarket.java +++ b/core/src/main/java/ru/trader/core/AbstractMarket.java @@ -7,6 +7,7 @@ public abstract class AbstractMarket implements Market { private final static Logger LOG = LoggerFactory.getLogger(AbstractMarket.class); private boolean change; + private boolean batch; protected abstract Place createPlace(String name, double x, double y, double z); protected abstract Group createGroup(String name, GROUP_TYPE type); @@ -17,6 +18,25 @@ public abstract class AbstractMarket implements Market { protected abstract void removePlace(Place place); protected abstract void addItem(Item item); protected abstract void removeItem(Item item); + protected void executeBatch(){} + + @Override + public void startBatch() { + LOG.debug("Start batch"); + batch = true; + } + + @Override + public final void doBatch() { + LOG.debug("End batch, updated"); + executeBatch(); + batch = false; + } + + @Override + public boolean isBatch() { + return batch; + } @Override public final void add(Place place){ diff --git a/core/src/main/java/ru/trader/core/Market.java b/core/src/main/java/ru/trader/core/Market.java index 280aacd..c2218b2 100644 --- a/core/src/main/java/ru/trader/core/Market.java +++ b/core/src/main/java/ru/trader/core/Market.java @@ -1,10 +1,17 @@ package ru.trader.core; -import java.util.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.Optional; import java.util.stream.Collectors; public interface Market { + void startBatch(); + void doBatch(); + boolean isBatch(); + void add(Place place); Place addPlace(String name, double x, double y, double z); void remove(Place place); @@ -109,6 +116,7 @@ public interface Market { } default void add(Market market){ + startBatch(); // add groups Collection groups = market.getGroups(); HashMap mapGroups = new HashMap<>(groups.size(), 0.9f); @@ -182,6 +190,7 @@ public interface Market { } } } + doBatch(); } default void clear(){ @@ -210,6 +219,7 @@ public interface Market { default void clear(boolean offers, boolean vendors, boolean places, boolean items, boolean groups){ + startBatch(); if (places){ Collection p = new ArrayList<>(get()); for (Place place : p) { @@ -238,6 +248,7 @@ public interface Market { remove(group); } } + doBatch(); } } diff --git a/core/src/main/java/ru/trader/store/simple/MarketDocHandler.java b/core/src/main/java/ru/trader/store/simple/MarketDocHandler.java index 9c9d862..56bc552 100644 --- a/core/src/main/java/ru/trader/store/simple/MarketDocHandler.java +++ b/core/src/main/java/ru/trader/store/simple/MarketDocHandler.java @@ -59,6 +59,7 @@ public class MarketDocHandler extends DefaultHandler { @Override public void startDocument() throws SAXException { world = new SimpleMarket(); + world.startBatch(); } @Override @@ -95,6 +96,7 @@ public class MarketDocHandler extends DefaultHandler { @Override public void endDocument() throws SAXException { + world.doBatch(); world.commit(); } diff --git a/core/src/main/java/ru/trader/store/simple/SimpleItemStat.java b/core/src/main/java/ru/trader/store/simple/SimpleItemStat.java index 90da4cf..c04b751 100644 --- a/core/src/main/java/ru/trader/store/simple/SimpleItemStat.java +++ b/core/src/main/java/ru/trader/store/simple/SimpleItemStat.java @@ -4,8 +4,12 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import ru.trader.core.*; -import java.util.*; +import java.util.Collection; +import java.util.Collections; +import java.util.DoubleSummaryStatistics; +import java.util.NavigableSet; import java.util.concurrent.ConcurrentSkipListSet; +import java.util.stream.Collectors; public class SimpleItemStat extends AbstractItemStat { private final static Logger LOG = LoggerFactory.getLogger(SimpleItemStat.class); @@ -36,6 +40,16 @@ public class SimpleItemStat extends AbstractItemStat { } } + synchronized void putAll(Collection offers){ + LOG.trace("Put offers to item stat {}", this); + if (this.offers.addAll(offers)){ + DoubleSummaryStatistics stat = this.offers.stream().collect(Collectors.summarizingDouble(Offer::getPrice)); + sum = stat.getSum(); + avg = stat.getAverage(); + LOG.trace("After this = {}", this); + } + } + synchronized void remove(Offer offer){ LOG.trace("Remove offer {} from item stat {}", offer, this); assert offer.hasType(type) && offer.hasItem(item); diff --git a/core/src/main/java/ru/trader/store/simple/SimpleMarket.java b/core/src/main/java/ru/trader/store/simple/SimpleMarket.java index 1d1a4ce..9524da6 100644 --- a/core/src/main/java/ru/trader/store/simple/SimpleMarket.java +++ b/core/src/main/java/ru/trader/store/simple/SimpleMarket.java @@ -5,6 +5,7 @@ import org.slf4j.LoggerFactory; import ru.trader.core.*; import java.util.*; +import java.util.stream.Stream; public class SimpleMarket extends AbstractMarket { private final static Logger LOG = LoggerFactory.getLogger(SimpleMarket.class); @@ -147,6 +148,7 @@ public class SimpleMarket extends AbstractMarket { @Override protected void onAdd(Vendor vendor) { + if (isBatch()) return; LOG.trace("Cached on add vendor {}", vendor); Collection offers = vendor.getAllSellOffers(); for (Offer offer : offers) { @@ -160,6 +162,7 @@ public class SimpleMarket extends AbstractMarket { @Override protected void onRemove(Vendor vendor) { + if (isBatch()) return; LOG.trace("Remove cache of vendor {}", vendor); Collection offers = vendor.getAllSellOffers(); for (Offer offer : offers) { @@ -173,12 +176,14 @@ public class SimpleMarket extends AbstractMarket { @Override protected void onAdd(Offer offer) { + if (isBatch()) return; LOG.trace("Cached on add offer {}", offer); put(getItemCache(offer.getType()), offer); } @Override protected void onRemove(Offer offer) { + if (isBatch()) return; LOG.trace("Remove cache of offer {}", offer); remove(getItemCache(offer.getType()), offer); } @@ -191,4 +196,48 @@ public class SimpleMarket extends AbstractMarket { systems.add(place); } + @Override + public void startBatch() { + super.startBatch(); + sellItems.clear(); + buyItems.clear(); + } + + @Override + protected void executeBatch() { + recreateCaches(); + super.executeBatch(); + } + + private void recreateCaches(){ + Map> sellOffers = new HashMap<>(); + Map> buyOffers = new HashMap<>(); + + systems.stream().flatMap(s -> s.get().stream()) + .flatMap(v -> Stream.concat(v.getAllSellOffers().stream(), v.getAllBuyOffers().stream())) + .forEach(o -> { + Map> m = o.getType() == OFFER_TYPE.SELL ? sellOffers : buyOffers; + Collection offers = m.get(o.getItem()); + if (offers == null){ + offers = new ArrayList<>(1000); + m.put(o.getItem(), offers); + } + offers.add(o); + }); + for (Iterator>> iterator = sellOffers.entrySet().iterator(); iterator.hasNext(); ) { + Map.Entry> entry = iterator.next(); + SimpleItemStat stat = new SimpleItemStat(entry.getKey(), OFFER_TYPE.SELL); + stat.putAll(entry.getValue()); + sellItems.put(stat.getItem(), stat); + iterator.remove(); + } + for (Iterator>> iterator = buyOffers.entrySet().iterator(); iterator.hasNext(); ) { + Map.Entry> entry = iterator.next(); + SimpleItemStat stat = new SimpleItemStat(entry.getKey(), OFFER_TYPE.BUY); + stat.putAll(entry.getValue()); + buyItems.put(stat.getItem(), stat); + iterator.remove(); + } + } + } diff --git a/utils/src/main/java/ru/trader/maddavo/Parser.java b/utils/src/main/java/ru/trader/maddavo/Parser.java index e2db8be..a733fb7 100644 --- a/utils/src/main/java/ru/trader/maddavo/Parser.java +++ b/utils/src/main/java/ru/trader/maddavo/Parser.java @@ -15,22 +15,23 @@ public class Parser { private boolean canceled; public void parseSystems(File file, Market market) throws IOException { - parseFile(file, new SystemHandler(market), 1); + parseFile(file, new SystemHandler(market), 1, market); } public void parseStations(File file, Market market) throws IOException { - parseFile(file, new StationHandler(market), 1); + parseFile(file, new StationHandler(market), 1, market); } public void parsePrices(File file, Market market) throws IOException { - parseFile(file, new OffersHandler(market, true), 0); + parseFile(file, new OffersHandler(market, true), 0, market); } - private void parseFile(File file, ParseHandler handler, int skip) throws IOException { + private void parseFile(File file, ParseHandler handler, int skip, Market market) throws IOException { canceled = false; + market.startBatch(); try (BufferedReader reader = new BufferedReader(new FileReader(file))) { int row = 0; String line; @@ -40,6 +41,8 @@ public class Parser { if (row <= skip) continue; handler.parse(line); } + } finally { + market.doBatch(); } }