From b2e502c9db0703ffce17997f747c408e8bb094b9 Mon Sep 17 00:00:00 2001 From: Rsl1122 Date: Tue, 1 Aug 2017 15:15:37 +0300 Subject: [PATCH 01/16] Update CertificateTutorial.md --- documentation/CertificateTutorial.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/documentation/CertificateTutorial.md b/documentation/CertificateTutorial.md index c4b8c8647..9a47720ff 100644 --- a/documentation/CertificateTutorial.md +++ b/documentation/CertificateTutorial.md @@ -62,7 +62,7 @@ Requirements: - A cloudflare account - A domain with full access -If you wish to bypass the security warning (not seeing that the connection isn’t private), you can use ![Cloudflare](https://www.cloudflare.com) +If you wish to bypass the security warning (not seeing that the connection isn’t private), you can use [Cloudflare](https://www.cloudflare.com) - Connect your domain to Cloudflare - Add a DNS record of the type “A” which points to your Server IP @@ -70,7 +70,7 @@ If you wish to bypass the security warning (not seeing that the connection isn Notes: If you only want to use HTTPS on the Analytics site, you can use the “Page Rules” -![Page Rules Tutorial](https://support.cloudflare.com/hc/en-us/articles/218411427-Page-Rules-Tutorial) +[Page Rules Tutorial](https://support.cloudflare.com/hc/en-us/articles/218411427-Page-Rules-Tutorial) It is recommend to activate “Automatic HTTPS Rewrites” under “Crypto” to be able to use http://LINK.TLD as well. This removes the need to write “https://” at the beginning of the address. From 8aea50ee6c082f612f89e6148a8932f25e2da28d Mon Sep 17 00:00:00 2001 From: Fuzzlemann Date: Tue, 1 Aug 2017 17:19:23 +0200 Subject: [PATCH 02/16] Fixes Issue #211 --- ...l => Maven__com_djrapitops_PlanPluginBridge_3_6_0.xml} | 8 ++++---- Plan/dependency-reduced-pom.xml | 5 ++++- .../plan/data/additional/PluginConfigSectionHandler.java | 5 +++++ .../plan/data/cache/GeolocationCacheHandler.java | 3 +-- 4 files changed, 14 insertions(+), 7 deletions(-) rename .idea/libraries/{Maven__com_djrapitops_PlanPluginBridge_3_5_0.xml => Maven__com_djrapitops_PlanPluginBridge_3_6_0.xml} (59%) diff --git a/.idea/libraries/Maven__com_djrapitops_PlanPluginBridge_3_5_0.xml b/.idea/libraries/Maven__com_djrapitops_PlanPluginBridge_3_6_0.xml similarity index 59% rename from .idea/libraries/Maven__com_djrapitops_PlanPluginBridge_3_5_0.xml rename to .idea/libraries/Maven__com_djrapitops_PlanPluginBridge_3_6_0.xml index ad29b78c8..6cdeb37a2 100644 --- a/.idea/libraries/Maven__com_djrapitops_PlanPluginBridge_3_5_0.xml +++ b/.idea/libraries/Maven__com_djrapitops_PlanPluginBridge_3_6_0.xml @@ -1,13 +1,13 @@ - + - + - + - + \ No newline at end of file diff --git a/Plan/dependency-reduced-pom.xml b/Plan/dependency-reduced-pom.xml index 8da93e303..d42de35ea 100644 --- a/Plan/dependency-reduced-pom.xml +++ b/Plan/dependency-reduced-pom.xml @@ -3,7 +3,7 @@ 4.0.0 com.djrapitops Plan - 3.5.5 + 3.6.0 ${basedir}/src clean package install @@ -12,6 +12,8 @@ . ${basedir}/src/main/resources + *.keystore + *.js *.yml *.html @@ -178,6 +180,7 @@ + java 1.8 UTF-8 1.8 diff --git a/Plan/src/main/java/com/djrapitops/plan/data/additional/PluginConfigSectionHandler.java b/Plan/src/main/java/com/djrapitops/plan/data/additional/PluginConfigSectionHandler.java index 0eed6db91..008a58c3e 100644 --- a/Plan/src/main/java/com/djrapitops/plan/data/additional/PluginConfigSectionHandler.java +++ b/Plan/src/main/java/com/djrapitops/plan/data/additional/PluginConfigSectionHandler.java @@ -25,6 +25,7 @@ public class PluginConfigSectionHandler { if (!section.contains(pluginName)) { return false; } + ConfigurationSection pluginSection = section.getConfigurationSection(pluginName); return pluginSection.contains(dataSource.getPlaceholder("")); } @@ -38,8 +39,10 @@ public class PluginConfigSectionHandler { ConfigurationSection section = getPluginsSection(); String pluginName = dataSource.getSourcePlugin(); String source = dataSource.placeholder; + section.addDefault(pluginName + ".Enabled", true); section.addDefault(pluginName + ".Data." + source, true); + FileConfiguration config = plan.getConfig(); config.set("Customization.Plugins", section); plan.saveConfig(); @@ -47,9 +50,11 @@ public class PluginConfigSectionHandler { public boolean isEnabled(PluginData dataSource) { ConfigurationSection section = getPluginsSection(); + String pluginName = dataSource.getSourcePlugin(); if (!section.getBoolean(pluginName + ".Enabled")) { return false; + } String source = dataSource.placeholder; return section.getBoolean(pluginName + ".Data." + source); diff --git a/Plan/src/main/java/com/djrapitops/plan/data/cache/GeolocationCacheHandler.java b/Plan/src/main/java/com/djrapitops/plan/data/cache/GeolocationCacheHandler.java index 5dcde6df6..9e54b1e49 100644 --- a/Plan/src/main/java/com/djrapitops/plan/data/cache/GeolocationCacheHandler.java +++ b/Plan/src/main/java/com/djrapitops/plan/data/cache/GeolocationCacheHandler.java @@ -16,7 +16,7 @@ import java.util.Map; *

* It caches all IPs with their matching country. *

- * This cache uses the Google Guava {@link Cache} and has a capacity of 10.000 entries. + * This cache uses the Google Guava {@link Cache}. * * @author Fuzzlemann * @since 3.5.5 @@ -31,7 +31,6 @@ public class GeolocationCacheHandler { } private static final Cache geolocationCache = CacheBuilder.newBuilder() - .maximumSize(10000) .build(); /** From 36574ec4e21eced99ac37c2a32da5677c4077810 Mon Sep 17 00:00:00 2001 From: Fuzzlemann Date: Tue, 1 Aug 2017 18:15:06 +0200 Subject: [PATCH 03/16] Add some more lambda --- .../plan/data/cache/InspectCacheHandler.java | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/Plan/src/main/java/com/djrapitops/plan/data/cache/InspectCacheHandler.java b/Plan/src/main/java/com/djrapitops/plan/data/cache/InspectCacheHandler.java index a8fb5162b..e475ae549 100644 --- a/Plan/src/main/java/com/djrapitops/plan/data/cache/InspectCacheHandler.java +++ b/Plan/src/main/java/com/djrapitops/plan/data/cache/InspectCacheHandler.java @@ -44,17 +44,14 @@ public class InspectCacheHandler { * @param uuid UUID of the player. */ public void cache(UUID uuid) { - DBCallableProcessor cacher = new DBCallableProcessor() { - @Override - public void process(UserData data) { - cache.put(uuid, new UserData(data)); - cacheTimes.put(uuid, MiscUtils.getTime()); - PageCacheHandler.cachePage("inspectPage: " + uuid.toString(), () -> new InspectPageResponse(Plan.getInstance().getUiServer().getDataReqHandler(), uuid)); - try { - ExportUtility.writeInspectHtml(data, ExportUtility.getPlayersFolder(ExportUtility.getFolder())); - } catch (IOException ex) { - Log.toLog(this.getClass().getName(), ex); - } + DBCallableProcessor cacher = data -> { + cache.put(uuid, new UserData(data)); + cacheTimes.put(uuid, MiscUtils.getTime()); + PageCacheHandler.cachePage("inspectPage: " + uuid.toString(), () -> new InspectPageResponse(Plan.getInstance().getUiServer().getDataReqHandler(), uuid)); + try { + ExportUtility.writeInspectHtml(data, ExportUtility.getPlayersFolder(ExportUtility.getFolder())); + } catch (IOException ex) { + Log.toLog(this.getClass().getName(), ex); } }; From f7941fd59fc2626a1188f72030a6b54ab6f25e55 Mon Sep 17 00:00:00 2001 From: Fuzzlemann Date: Tue, 1 Aug 2017 19:38:43 +0200 Subject: [PATCH 04/16] Adds more visibility to the code --- .../utilities/analysis/AnalysisUtils.java | 48 ++++++++++--------- 1 file changed, 26 insertions(+), 22 deletions(-) diff --git a/Plan/src/main/java/com/djrapitops/plan/utilities/analysis/AnalysisUtils.java b/Plan/src/main/java/com/djrapitops/plan/utilities/analysis/AnalysisUtils.java index 5a2bc1c1e..ece30fa69 100644 --- a/Plan/src/main/java/com/djrapitops/plan/utilities/analysis/AnalysisUtils.java +++ b/Plan/src/main/java/com/djrapitops/plan/utilities/analysis/AnalysisUtils.java @@ -162,18 +162,19 @@ public class AnalysisUtils { * @return */ public static String getBooleanPercentage(AnalysisType analysisType, PluginData source, List uuids) { - if (analysisType == AnalysisType.BOOLEAN_PERCENTAGE) { - try { - List tempList = getCorrectValues(uuids, source) - .map(value -> (boolean) value) - .collect(Collectors.toList()); - long count = tempList.stream().filter(value -> value).count(); - return source.parseContainer(analysisType.getModifier(), (((double) count / tempList.size()) * 100) + "%"); - } catch (Exception | NoClassDefFoundError | NoSuchFieldError e) { - return logPluginDataCausedError(source, e); - } + if (analysisType != AnalysisType.BOOLEAN_PERCENTAGE) { + return source.parseContainer("Err ", "Wrong Analysistype specified: " + analysisType.name()); + } + + try { + List tempList = getCorrectValues(uuids, source) + .map(value -> (boolean) value) + .collect(Collectors.toList()); + long count = tempList.stream().filter(value -> value).count(); + return source.parseContainer(analysisType.getModifier(), (((double) count / tempList.size()) * 100) + "%"); + } catch (Exception | NoClassDefFoundError | NoSuchFieldError e) { + return logPluginDataCausedError(source, e); } - return source.parseContainer("Err ", "Wrong Analysistype specified: " + analysisType.name()); } /** @@ -183,18 +184,19 @@ public class AnalysisUtils { * @return */ public static String getBooleanTotal(AnalysisType analysisType, PluginData source, List uuids) { - if (analysisType == AnalysisType.BOOLEAN_TOTAL) { - try { - List tempList = getCorrectValues(uuids, source) - .map(value -> (boolean) value) - .collect(Collectors.toList()); - long count = tempList.stream().filter(value -> value).count(); - return source.parseContainer(analysisType.getModifier(), count + " / " + tempList.size()); - } catch (Exception e) { - return logPluginDataCausedError(source, e); - } + if (analysisType != AnalysisType.BOOLEAN_TOTAL) { + return source.parseContainer("Err ", "Wrong Analysistype specified: " + analysisType.name()); + } + + try { + List tempList = getCorrectValues(uuids, source) + .map(value -> (boolean) value) + .collect(Collectors.toList()); + long count = tempList.stream().filter(value -> value).count(); + return source.parseContainer(analysisType.getModifier(), count + " / " + tempList.size()); + } catch (Exception e) { + return logPluginDataCausedError(source, e); } - return source.parseContainer("Err ", "Wrong Analysistype specified: " + analysisType.name()); } private static String logPluginDataCausedError(PluginData source, Throwable e) { @@ -214,6 +216,7 @@ public class AnalysisUtils { public static int getUniqueJoins(Map> sessions, long scale) { long now = MiscUtils.getTime(); long nowMinusScale = now - scale; + Set uniqueJoins = new HashSet<>(); sessions.forEach((uuid, s) -> s.stream() @@ -221,6 +224,7 @@ public class AnalysisUtils { .map(session -> uuid) .forEach(uniqueJoins::add) ); + return uniqueJoins.size(); } From 15abeb294dda17d4ba0a8959dc7bb10138d5383f Mon Sep 17 00:00:00 2001 From: Fuzzlemann Date: Tue, 1 Aug 2017 20:34:23 +0200 Subject: [PATCH 05/16] Inline variables Convert some things to Method Reference Add randomEnum(Class) --- .../com/djrapitops/plan/database/databases/SQLDB.java | 2 +- Plan/src/test/java/utils/RandomData.java | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/Plan/src/main/java/com/djrapitops/plan/database/databases/SQLDB.java b/Plan/src/main/java/com/djrapitops/plan/database/databases/SQLDB.java index c6a824273..b78955a41 100644 --- a/Plan/src/main/java/com/djrapitops/plan/database/databases/SQLDB.java +++ b/Plan/src/main/java/com/djrapitops/plan/database/databases/SQLDB.java @@ -450,7 +450,7 @@ public abstract class SQLDB extends Database { .map(UserData::getWorldTimes) .map(WorldTimes::getTimes) .map(Map::keySet) - .flatMap(keySet -> keySet.stream()) + .flatMap(Collection::stream) .distinct() .collect(Collectors.toList()); for (Map.Entry entrySet : userDatas.entrySet()) { diff --git a/Plan/src/test/java/utils/RandomData.java b/Plan/src/test/java/utils/RandomData.java index 8b627dfbf..82cfd194a 100644 --- a/Plan/src/test/java/utils/RandomData.java +++ b/Plan/src/test/java/utils/RandomData.java @@ -34,8 +34,7 @@ public class RandomData { * Random enough. */ public static String randomString(int size) { - String randomString = UUID.randomUUID().toString().replaceAll("-", "").substring(0, size); - return randomString; + return UUID.randomUUID().toString().replaceAll("-", "").substring(0, size); } public static List randomWebUsers() throws PassEncryptUtil.CannotPerformOperationException { @@ -75,7 +74,7 @@ public class RandomData { public static List randomHandlingInfo() { List test = new ArrayList<>(); for (int i = 0; i < 20; i++) { - test.add(new HandlingInfo(UUID.randomUUID(), InfoType.OTHER, r.nextLong()) { + test.add(new HandlingInfo(UUID.randomUUID(), randomEnum(InfoType.class), r.nextLong()) { @Override public boolean process(UserData uData) { return false; @@ -85,5 +84,8 @@ public class RandomData { return test; } - + public static T randomEnum(Class clazz) { + int x = r.nextInt(clazz.getEnumConstants().length); + return clazz.getEnumConstants()[x]; + } } From 05c893b35c881d1bb67dcc533a804719b5881354 Mon Sep 17 00:00:00 2001 From: Fuzzlemann Date: Tue, 1 Aug 2017 20:38:43 +0200 Subject: [PATCH 06/16] Replace UUID.randomUUID() to RandomStringUtils.random(size) --- Plan/src/test/java/utils/RandomData.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Plan/src/test/java/utils/RandomData.java b/Plan/src/test/java/utils/RandomData.java index 82cfd194a..5646c7256 100644 --- a/Plan/src/test/java/utils/RandomData.java +++ b/Plan/src/test/java/utils/RandomData.java @@ -9,6 +9,7 @@ import main.java.com.djrapitops.plan.data.handling.info.InfoType; import main.java.com.djrapitops.plan.database.tables.GMTimesTable; import main.java.com.djrapitops.plan.utilities.PassEncryptUtil; import main.java.com.djrapitops.plan.utilities.analysis.Point; +import org.apache.commons.lang.RandomStringUtils; import java.util.ArrayList; import java.util.List; @@ -34,7 +35,7 @@ public class RandomData { * Random enough. */ public static String randomString(int size) { - return UUID.randomUUID().toString().replaceAll("-", "").substring(0, size); + return RandomStringUtils.random(size); } public static List randomWebUsers() throws PassEncryptUtil.CannotPerformOperationException { From 53cd34978b7b47a0db2e395a7ed8b68c767955da Mon Sep 17 00:00:00 2001 From: Fuzzlemann Date: Tue, 1 Aug 2017 22:28:58 +0200 Subject: [PATCH 07/16] Adds a KillDataComparator to list the KillData correctly and in the right order --- .../plan/ui/html/tables/KillsTableCreator.java | 6 ++++++ .../comparators/KillDataComparator.java | 17 +++++++++++++++++ 2 files changed, 23 insertions(+) create mode 100644 Plan/src/main/java/com/djrapitops/plan/utilities/comparators/KillDataComparator.java diff --git a/Plan/src/main/java/com/djrapitops/plan/ui/html/tables/KillsTableCreator.java b/Plan/src/main/java/com/djrapitops/plan/ui/html/tables/KillsTableCreator.java index 8cc79dbeb..c3a7ca489 100644 --- a/Plan/src/main/java/com/djrapitops/plan/ui/html/tables/KillsTableCreator.java +++ b/Plan/src/main/java/com/djrapitops/plan/ui/html/tables/KillsTableCreator.java @@ -6,7 +6,9 @@ import main.java.com.djrapitops.plan.data.KillData; import main.java.com.djrapitops.plan.ui.html.Html; import main.java.com.djrapitops.plan.utilities.FormatUtils; import main.java.com.djrapitops.plan.utilities.HtmlUtils; +import main.java.com.djrapitops.plan.utilities.comparators.KillDataComparator; +import java.util.Collections; import java.util.List; /** @@ -32,6 +34,10 @@ public class KillsTableCreator { html.append(Html.TABLELINE_3.parse(Html.KILLDATA_NONE.parse(), "", "")); } else { int i = 0; + + killData.sort(new KillDataComparator()); + Collections.reverse(killData); + for (KillData kill : killData) { if (i >= 20) { break; diff --git a/Plan/src/main/java/com/djrapitops/plan/utilities/comparators/KillDataComparator.java b/Plan/src/main/java/com/djrapitops/plan/utilities/comparators/KillDataComparator.java new file mode 100644 index 000000000..42bb5c877 --- /dev/null +++ b/Plan/src/main/java/com/djrapitops/plan/utilities/comparators/KillDataComparator.java @@ -0,0 +1,17 @@ +package main.java.com.djrapitops.plan.utilities.comparators; + +import main.java.com.djrapitops.plan.data.KillData; + +import java.util.Comparator; + +/** + * @author Fuzzlemann + */ +public class KillDataComparator implements Comparator { + + @Override + public int compare(KillData o1, KillData o2) { + return Long.compare(o1.getDate(), o2.getDate()); + } + +} From f705877cbcf28d4cc07432ac66aeb576f75495d6 Mon Sep 17 00:00:00 2001 From: Fuzzlemann Date: Wed, 2 Aug 2017 01:30:32 +0200 Subject: [PATCH 08/16] Change TimeKeeper.toString() Simplifies TimeKeeper.equals() --- .../djrapitops/plan/data/time/TimeKeeper.java | 48 ++++++++++--------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/Plan/src/main/java/com/djrapitops/plan/data/time/TimeKeeper.java b/Plan/src/main/java/com/djrapitops/plan/data/time/TimeKeeper.java index b17c28c41..80e04fcbd 100644 --- a/Plan/src/main/java/com/djrapitops/plan/data/time/TimeKeeper.java +++ b/Plan/src/main/java/com/djrapitops/plan/data/time/TimeKeeper.java @@ -117,26 +117,6 @@ public abstract class TimeKeeper { return times; } - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - TimeKeeper that = (TimeKeeper) o; - - if (lastStateChange != that.lastStateChange) return false; - if (times != null ? !times.equals(that.times) : that.times != null) return false; - return state != null ? state.equals(that.state) : that.state == null; - } - - @Override - public int hashCode() { - int result = times != null ? times.hashCode() : 0; - result = 31 * result + (state != null ? state.hashCode() : 0); - result = 31 * result + (int) (lastStateChange ^ (lastStateChange >>> 32)); - return result; - } - public void setState(String state) { this.state = state; } @@ -153,10 +133,32 @@ public abstract class TimeKeeper { return lastStateChange; } + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + TimeKeeper that = (TimeKeeper) o; + + return lastStateChange == that.lastStateChange && + times != null ? times.equals(that.times) : that.times == null + && state != null ? state.equals(that.state) : that.state == null; + } + + @Override + public int hashCode() { + int result = times != null ? times.hashCode() : 0; + result = 31 * result + (state != null ? state.hashCode() : 0); + result = 31 * result + (int) (lastStateChange ^ (lastStateChange >>> 32)); + return result; + } + @Override public String toString() { - return "times:" + times + - ",state:" + state + - ",lastStateChange:" + lastStateChange; + return "TimeKeeper{" + + "times=" + times + + ", state='" + state + '\'' + + ", lastStateChange=" + lastStateChange + + '}'; } } \ No newline at end of file From af91f25827c3c8b586d3782c2b9c5ccfba18029f Mon Sep 17 00:00:00 2001 From: Fuzzlemann Date: Wed, 2 Aug 2017 18:46:59 +0200 Subject: [PATCH 09/16] Fix 'i' now counting at WorldPieCreator.createSeriesData(Map) Make the creation of offline players at the import parallel (before: 68608 ms, now: 20320 ms) --- .../data/handling/importing/Importer.java | 44 ++++++++++++++----- .../plan/ui/html/graphs/WorldPieCreator.java | 3 ++ .../plan/database/DatabaseTest.java | 11 ++--- 3 files changed, 41 insertions(+), 17 deletions(-) diff --git a/Plan/src/main/java/com/djrapitops/plan/data/handling/importing/Importer.java b/Plan/src/main/java/com/djrapitops/plan/data/handling/importing/Importer.java index 86a124843..29fe13671 100644 --- a/Plan/src/main/java/com/djrapitops/plan/data/handling/importing/Importer.java +++ b/Plan/src/main/java/com/djrapitops/plan/data/handling/importing/Importer.java @@ -2,6 +2,7 @@ package main.java.com.djrapitops.plan.data.handling.importing; import com.djrapitops.plugin.utilities.player.Fetch; import com.djrapitops.plugin.utilities.player.IOfflinePlayer; +import com.djrapitops.plugin.utilities.status.ProcessStatus; import main.java.com.djrapitops.plan.Log; import main.java.com.djrapitops.plan.Plan; import main.java.com.djrapitops.plan.data.UserData; @@ -14,6 +15,7 @@ import main.java.com.djrapitops.plan.utilities.NewPlayerCreator; import java.sql.SQLException; import java.util.*; +import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Function; import java.util.stream.Collectors; @@ -58,11 +60,16 @@ public abstract class Importer { public boolean importData(Collection uuids, String... args) { Plan plan = Plan.getInstance(); plan.getAnalysisCache().disableAnalysisTemporarily(); + try { String processName = "Import, " + getClass().getSimpleName(); - plan.processStatus().startExecution(processName); + + ProcessStatus processStatus = plan.processStatus(); DataCacheHandler handler = plan.getHandler(); Database db = plan.getDB(); + + processStatus.startExecution(processName); + Set saved; try { saved = db.getSavedUUIDs(); @@ -70,30 +77,47 @@ public abstract class Importer { Log.toLog(this.getClass().getName(), ex); return false; } + List unSaved = new ArrayList<>(uuids); unSaved.removeAll(saved); + String createUserObjects = "Creating new UserData objects for: " + unSaved.size(); - plan.processStatus().setStatus(processName, createUserObjects); + processStatus.setStatus(processName, createUserObjects); + Map offlinePlayers = Fetch.getIOfflinePlayers().stream().collect(Collectors.toMap(IOfflinePlayer::getUuid, Function.identity())); + Benchmark.start(createUserObjects); - List offlineP = unSaved.stream().map(offlinePlayers::get).collect(Collectors.toList()); + List newUsers = new ArrayList<>(); - for (IOfflinePlayer p : offlineP) { - UserData newPlayer = NewPlayerCreator.createNewOfflinePlayer(p); - newPlayer.setLastPlayed(newPlayer.getRegistered()); - newUsers.add(newPlayer); - plan.processStatus().setStatus(processName, "Creating new UserData objects: " + newUsers.size() + "/" + unSaved.size()); - } + List offlineP = unSaved + .stream() + .map(offlinePlayers::get) + .collect(Collectors.toList()); + + AtomicInteger currentUser = new AtomicInteger(0); + int amount = unSaved.size(); + + offlineP.parallelStream() + .map(NewPlayerCreator::createNewOfflinePlayer) + .forEach(newPlayer -> { + newPlayer.setLastPlayed(newPlayer.getRegistered()); + newUsers.add(newPlayer); + processStatus.setStatus(processName, "Creating new UserData objects: " + currentUser.addAndGet(1) + "/" + amount); + }); + Benchmark.stop(createUserObjects); - plan.processStatus().setStatus(processName, "Save new UserData objects (" + unSaved.size() + ")"); + processStatus.setStatus(processName, "Save new UserData objects (" + unSaved.size() + ")"); + try { plan.getDB().saveMultipleUserData(newUsers); } catch (SQLException ex) { Log.toLog(this.getClass().getName(), ex); } + for (UUID uuid : uuids) { handler.addToPool(importData(uuid, args)); } + plan.processStatus().finishExecution(processName); } finally { plan.getAnalysisCache().enableAnalysis(); diff --git a/Plan/src/main/java/com/djrapitops/plan/ui/html/graphs/WorldPieCreator.java b/Plan/src/main/java/com/djrapitops/plan/ui/html/graphs/WorldPieCreator.java index c70a9b4b6..9c6f2b59f 100644 --- a/Plan/src/main/java/com/djrapitops/plan/ui/html/graphs/WorldPieCreator.java +++ b/Plan/src/main/java/com/djrapitops/plan/ui/html/graphs/WorldPieCreator.java @@ -15,13 +15,16 @@ public class WorldPieCreator { for (Map.Entry world : worldTimes.entrySet()) { arrayBuilder.append("{name:'").append(world.getKey()) .append("',y:").append(world.getValue()); + if (i == 1) { arrayBuilder.append(", sliced: true, selected: true"); } + arrayBuilder.append("}"); if (i < size - 1) { arrayBuilder.append(","); } + i++; } arrayBuilder.append("]"); return arrayBuilder.toString(); diff --git a/Plan/src/test/java/main/java/com/djrapitops/plan/database/DatabaseTest.java b/Plan/src/test/java/main/java/com/djrapitops/plan/database/DatabaseTest.java index e4e8274b9..106618a43 100644 --- a/Plan/src/test/java/main/java/com/djrapitops/plan/database/DatabaseTest.java +++ b/Plan/src/test/java/main/java/com/djrapitops/plan/database/DatabaseTest.java @@ -235,13 +235,10 @@ public class DatabaseTest { db.saveUserData(data); data.addNickname("TestUpdateForSave"); db.saveUserData(data); - DBCallableProcessor process = new DBCallableProcessor() { - @Override - public void process(UserData d) { - System.out.println("\nOriginal: " + data); - System.out.println("Database: " + d); - assertTrue("Not Equals", data.equals(d)); - } + DBCallableProcessor process = d -> { + System.out.println("\nOriginal: " + data); + System.out.println("Database: " + d); + assertTrue("Not Equals", data.equals(d)); }; db.giveUserDataToProcessors(data.getUuid(), process); } From 3559b66b558820ff8b8e6227d0f0f538f9453f31 Mon Sep 17 00:00:00 2001 From: Fuzzlemann Date: Wed, 2 Aug 2017 18:59:55 +0200 Subject: [PATCH 10/16] Fixes Issue #217 --- .../plan/data/listeners/PlanDeathEventListener.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Plan/src/main/java/com/djrapitops/plan/data/listeners/PlanDeathEventListener.java b/Plan/src/main/java/com/djrapitops/plan/data/listeners/PlanDeathEventListener.java index 9c25f8fd9..dc7f59cce 100644 --- a/Plan/src/main/java/com/djrapitops/plan/data/listeners/PlanDeathEventListener.java +++ b/Plan/src/main/java/com/djrapitops/plan/data/listeners/PlanDeathEventListener.java @@ -60,10 +60,10 @@ public class PlanDeathEventListener implements Listener { Material itemInHand; try { itemInHand = killer.getInventory().getItemInMainHand().getType(); - } catch (Exception e) { + } catch (NoSuchMethodError e) { try { itemInHand = killer.getInventory().getItemInHand().getType(); // Support for non dual wielding versions. - } catch (Exception e2) { + } catch (Error e2) { itemInHand = Material.AIR; } } From deb2feed33c1fef2dcec6f1ade3801bd5b693f1a Mon Sep 17 00:00:00 2001 From: Rsl1122 Date: Wed, 2 Aug 2017 22:53:23 +0300 Subject: [PATCH 11/16] Create license.yml --- Plan/src/main/resources/license.yml | 124 ++++++++++++++++++++++++++++ 1 file changed, 124 insertions(+) create mode 100644 Plan/src/main/resources/license.yml diff --git a/Plan/src/main/resources/license.yml b/Plan/src/main/resources/license.yml new file mode 100644 index 000000000..9a4cb50c0 --- /dev/null +++ b/Plan/src/main/resources/license.yml @@ -0,0 +1,124 @@ +Player Analytics (Plan) License v.2.0.0 + +Copyright (C) Risto Lahtela / Rsl1122 +Everyone is permitted to copy and distribute verbatim copies +of this license document, but changing it is not allowed. + +0. Definitions + +With 'Software' this license refers to the following: +Player Analytics (Plan) Bukkit plugin +(https://github.com/Rsl1122/Plan-PlayerAnalytics), +Modified versions the software +& the source code that is publicly available on Plan’s GitHub repository. +This does not include the Markdown (.md) files available in the repository. + +‘Java Runtime Environment’ +A Runtime Environment which allows Java Bytecode to run, mostly Oracle JRE. +But this also includes other Runtime Environments like OpenJDK / JRE (http://openjdk.java.net) + +‘Java Virtual Machine’ +The virtual machine included in the Java Runtime Environment. + +‘GitHub’ +(https://www.github.com) + +‘Minecraft’ + A game by Mojang AB (https://www.mojang.com). +(https://www.minecraft.net) + +‘Minecraft Server software’ +The software to operate a server for Minecraft, mostly Bukkit or its forks. +Original Minecraft Server independent software like Glowstone (https://www.glowstone.net) are falling under this definition as well. + +‘Plan’s Spigot plugin page’ +(https://www.spigotmc.org/resources/plan-player-analytics.32536) + +‘Minecraft EULA’ +(https://help.mojang.com/customer/en/portal/articles/1590522-minecraft-commercial-use) + +‘Plan’s GitHub repository’ +(https://www.github.com/Rsl1122/Plan-PlayerAnalytics) + +‘Fork’ +Refers to a copy made of Plan’s GitHub repository which is using the Fork System of GitHub. + +'End User' +A person running the software on a Java Virtual Machine environment. + +‘Original Source Code’ +Refers to the source code that is available on Plan’s GitHub repository. + +‘Original Author’ +Rsl1122 / Risto Lahtela + +1. Use & Distribution + + +This license grants the end user the rights to run the software & make modifications to its HTML components, Java Keystore, locale files +& configuration files in the scope that the plugin allows without decompiling the .class files inside the jar. + +Distribution and modification of source code is allowed under the circumstances that such code is publicly available, the original +author is mentioned & the original author is made aware of the modified code. A fork counts as a notification about modified code. +A removal of the Fork display is not allowed under any circumstances except if the original author grants a written permission for +this exact action. + +All versions downloaded from the Plan’s Spigot plugin page & .jar packages compiled +from the original source code can be used as a dependency and only as a dependency in another project, commercial or not. + + +You may not distribute or sell modified, or unmodified versions of compiled source code without a direct written permission of +the original author. +Such permissions are granted separately for Distribution & Selling. + +When distributing a modified, or unmodified copy of the software you grant the right for the original author to post a takedown +notice, upon which the modified software must be removed from distribution. +If the conditions of the license are breached, the software breaking the license must be removed from distribution. + +2. Commercial use + +When using parts of the software with Minecraft Server software, the use must comply with EULA provided by Mojang. +If a project requires the end user to obtain a copy of the software, a link must be provided to obtain the software from the author +or Spigot. + +3. Modification + + +Modification on the code is allowed, as long as the section 1 of this license is not violated. + +4. Usage of part of the code in another program + +This part of the license is about copying code directly from the source code or decompiled .class files. + +Usage of a part of code in another non-commercial program is allowed if following conditions apply: + a. The program falls under this license + b. This license is accepted by the person using the code inside their program + c. The use does not break the conditions of this license + d. The use is not a clear copyright infringement + OR + a. It can be proven that it is possible to come to the same solution independently in a reasonable time frame of one hour with + no prior access to the software. + OR + a. The original author grants written permission for the use. + +5. Revised versions of this License + +The author may publish revised and/or new versions of this License from time to time. Such new versions will be similar in spirit +to the old version, but may differ in detail to address new problems or concerns. + +6. Disclaimer of Warranty. + +THERE IS NO WARRANTY FOR THE SOFTWARE, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE SOFTWARE "AS IS" +WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY +AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE SOFTWARE IS WITH YOU. +SHOULD THE SOFTWARE PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + +7. Limitation of Liability. + +IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS THE SOFTWARE AS PERMITTED ABOVE, BE LIABLE TO YOU FOR +DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE SOFTWARE +(INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE +OF THE SOFTWARE TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH +DAMAGES. From 4afd13f9f8f6a909f4245428b4f1bad8d5df2796 Mon Sep 17 00:00:00 2001 From: Rsl1122 Date: Wed, 2 Aug 2017 22:53:57 +0300 Subject: [PATCH 12/16] Update licence.yml --- Plan/src/main/resources/licence.yml | 108 +--------------------------- 1 file changed, 1 insertion(+), 107 deletions(-) diff --git a/Plan/src/main/resources/licence.yml b/Plan/src/main/resources/licence.yml index 6bfe42d07..8f18c5649 100644 --- a/Plan/src/main/resources/licence.yml +++ b/Plan/src/main/resources/licence.yml @@ -1,107 +1 @@ -Player Analytics (Plan) Licence v.1.0.1 - -Copyright (C) Risto Lahtela / Rsl1122 -Everyone is permitted to copy and distribute verbatim copies -of this license document, but changing it is not allowed. - - -0. Definitions - -With 'software' this licence refers to the following: -Player Analytics (Plan) Bukkit plugin -Player Analytics Lite (PlanLite) Bukkit plugin -(https://github.com/Rsl1122/Plan-PlayerAnalytics) -Player Demographics (Plade) Bukkit plugin -(https://github.com/Rsl1122/Plan-Player-Demographics) -& the source code of these that is publicly available. - -'visual aspect of software' -The messages sent with Bukkit API to Player/Console -Html pages & the included styles -Images & Videos found on the plugin page - -Additional 'resources' -Provided .yml .html .txt and image files inside the software. - -'code modifyer' -A person modifying the code/resources of the software. - -'user' -A person running the software - - -1. Use & Distribution - -This licence grants the end user the rights to run the software & modify it to a certain extent. (detailed in section 3. Modification) -All versions downloaded from Spigot (https://www.spigotmc.org/resources/authors/rsl1122.122894/) & .jar packages compiled -from the source code can be used as a dependency in another project, commercial or not. - -You may not sell the software or modified versions of this software. - -Modified software can only be distributed if ALL of the following conditions apply: - a. the original author and the original work are clearly mentioned & linked in the distribution method or when using the software - b. Modified software does not contain harmful code to the user. - c. Modified software is free & the modified source code is publicly available. - d. Modified software is published under this licence - e. The end user is provided a copy of this licence - - When distributing a modified copy of the software you grant the right for the original author to post a takedown notice, upon which - the modified source code and modified software must be removed from distribution. - -2. Commercial use - -When using parts of the software with 'Minecraft Server software', the use must comply with EULA provided by Mojang (https://help.mojang.com/customer/en/portal/articles/1590522-minecraft-commercial-use) -If a project requires end user to obtain a copy of the software, a link must be provided to obtain the software from the author / Spigot. - - -3. Modification - -The code & resources can be modified & run if the following conditions apply: - a. The modified version is used only by the modifyer (Person modifying the resources/software) & not distributed/sold. - The modifyer may transfer his right to use the software to another person in following cases: - a1: Modifyer was paid to modify part of the software for customization of visual appearance of the software or fix Exceptions & Errors caused - by updates to Minecraft Server software. - - -4. Usage of part of code in another software - -This part (4) of the licence does not include the usage of the API of the plugin, -or adding the compiled code as provided dependency. Such use is allowed. - -Usage of a part of code in another software is allowed if following conditions apply: - a. The software falls under this licence - b. This licence is accepted by the person using the code inside their software - c. The use is not a clear copyright infringement - OR - a. It can be proven that it is possible to come to the same solution independently in a resonable time frame of one hour - with no prior access to the software. - - -5. Revised Versions of this Licence - -The author may publish revised and/or new versions of -this License from time to time. Such new versions will be similar in spirit to the present version, -but may differ in detail to address new problems or concerns. - -6. Disclaimer of Warranty. - - THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY -APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT -HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY -OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, -THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM -IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF -ALL NECESSARY SERVICING, REPAIR OR CORRECTION. - -7. Limitation of Liability. - - IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING -WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS -THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY -GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE -USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF -DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD -PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), -EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF -SUCH DAMAGES. \ No newline at end of file +Moved to https://github.com/Rsl1122/Plan-PlayerAnalytics/edit/master/Plan/src/main/resources/license.yml From 5be0042823165325c8e3c0c2dd939c86a7e7d755 Mon Sep 17 00:00:00 2001 From: Rsl1122 Date: Wed, 2 Aug 2017 22:54:16 +0300 Subject: [PATCH 13/16] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 87e4b1707..4d692bf37 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ Originally the plugin only displayed data of other plugins, but now it gathers i ### Links - [Spigot, Resource page](https://www.spigotmc.org/resources/plan-player-analytics.32536/) - [Issues & Suggestions](https://github.com/Rsl1122/Plan-PlayerAnalytics/issues) -- [Licence](https://github.com/Rsl1122/Plan-PlayerAnalytics/blob/master/Plan/src/main/resources/licence.yml) +- [License](https://github.com/Rsl1122/Plan-PlayerAnalytics/blob/master/Plan/src/main/resources/license.yml) ## Documentation - [Javadocs](https://rsl1122.github.io/Plan-PlayerAnalytics/) (OutDated) From 52cec5912b39946d4943d7450a7f98af223530cb Mon Sep 17 00:00:00 2001 From: Fuzzlemann Date: Wed, 2 Aug 2017 22:22:02 +0200 Subject: [PATCH 14/16] Replace unSaved.size() with introduced value amount --- .../com/djrapitops/plan/data/handling/importing/Importer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Plan/src/main/java/com/djrapitops/plan/data/handling/importing/Importer.java b/Plan/src/main/java/com/djrapitops/plan/data/handling/importing/Importer.java index 29fe13671..cf5ecd8e1 100644 --- a/Plan/src/main/java/com/djrapitops/plan/data/handling/importing/Importer.java +++ b/Plan/src/main/java/com/djrapitops/plan/data/handling/importing/Importer.java @@ -106,7 +106,7 @@ public abstract class Importer { }); Benchmark.stop(createUserObjects); - processStatus.setStatus(processName, "Save new UserData objects (" + unSaved.size() + ")"); + processStatus.setStatus(processName, "Save new UserData objects (" + amount + ")"); try { plan.getDB().saveMultipleUserData(newUsers); From dedb95766a5fbda7bdaaf9659d89b993cf53c623 Mon Sep 17 00:00:00 2001 From: Fuzzlemann Date: Wed, 2 Aug 2017 23:47:43 +0200 Subject: [PATCH 15/16] Only set the status every 5% -> Performance Improvement of 17s; previous: 20s, now: 4s. --- .../plan/data/handling/importing/Importer.java | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/Plan/src/main/java/com/djrapitops/plan/data/handling/importing/Importer.java b/Plan/src/main/java/com/djrapitops/plan/data/handling/importing/Importer.java index cf5ecd8e1..3812ab806 100644 --- a/Plan/src/main/java/com/djrapitops/plan/data/handling/importing/Importer.java +++ b/Plan/src/main/java/com/djrapitops/plan/data/handling/importing/Importer.java @@ -18,6 +18,7 @@ import java.util.*; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Function; import java.util.stream.Collectors; +import java.util.stream.IntStream; /** * Abstract class used for importing data from other plugins. @@ -81,7 +82,9 @@ public abstract class Importer { List unSaved = new ArrayList<>(uuids); unSaved.removeAll(saved); - String createUserObjects = "Creating new UserData objects for: " + unSaved.size(); + int amount = unSaved.size(); + + String createUserObjects = "Creating " + amount + " new UserData objects"; processStatus.setStatus(processName, createUserObjects); Map offlinePlayers = Fetch.getIOfflinePlayers().stream().collect(Collectors.toMap(IOfflinePlayer::getUuid, Function.identity())); @@ -95,14 +98,23 @@ public abstract class Importer { .collect(Collectors.toList()); AtomicInteger currentUser = new AtomicInteger(0); - int amount = unSaved.size(); + AtomicInteger currentPercent = new AtomicInteger(0); + + int fivePercent = amount / 20; + + //Using Set because of better Collection#contains() performance + Set milestones = IntStream.rangeClosed(1, 20) + .mapToObj(i -> i * fivePercent) + .collect(Collectors.toSet()); offlineP.parallelStream() .map(NewPlayerCreator::createNewOfflinePlayer) .forEach(newPlayer -> { newPlayer.setLastPlayed(newPlayer.getRegistered()); newUsers.add(newPlayer); - processStatus.setStatus(processName, "Creating new UserData objects: " + currentUser.addAndGet(1) + "/" + amount); + if (milestones.contains(currentUser.incrementAndGet())) { + processStatus.setStatus(processName, "Creating new UserData objects: " + currentPercent.addAndGet(5) + "%"); + } }); Benchmark.stop(createUserObjects); From 62e73ffe1c6400ee95bbb5ce1343f09be72bd8a2 Mon Sep 17 00:00:00 2001 From: Fuzzlemann Date: Thu, 3 Aug 2017 03:25:48 +0200 Subject: [PATCH 16/16] Adds the flag "rewriteBatchedStatements=true" to the MySQL JDBC connection URL to pack batches into one Query Fixes NullPointerException at saveMultipleUserData(Collection) Improves the save... methods Makes Batch Insertion parallel Removed NoSuchFieldError where it's not going to happen Modifies GM Times to update / insert in batches (previous: 520977ms (521s); after: 495115ms (495s) --- .../com/djrapitops/plan/database/DBUtils.java | 22 +++ .../plan/database/databases/MySQLDB.java | 4 +- .../plan/database/databases/SQLDB.java | 20 ++- .../plan/database/tables/CommandUseTable.java | 1 + .../plan/database/tables/GMTimesTable.java | 169 ++++++++++++------ .../plan/database/tables/IPsTable.java | 1 + .../plan/database/tables/SessionsTable.java | 58 +++--- .../plan/database/tables/TPSTable.java | 29 +-- .../plan/database/tables/UsersTable.java | 38 ++-- 9 files changed, 237 insertions(+), 105 deletions(-) diff --git a/Plan/src/main/java/com/djrapitops/plan/database/DBUtils.java b/Plan/src/main/java/com/djrapitops/plan/database/DBUtils.java index 8548c3136..ec9426305 100644 --- a/Plan/src/main/java/com/djrapitops/plan/database/DBUtils.java +++ b/Plan/src/main/java/com/djrapitops/plan/database/DBUtils.java @@ -72,6 +72,7 @@ public class DBUtils { if (wrappedBatches.size() <= j) { wrappedBatches.add(new ArrayList<>()); } + wrappedBatches.get(j).add(new Container<>(object, entry.getKey())); i++; if (i % BATCH_SIZE == 0) { @@ -81,4 +82,25 @@ public class DBUtils { } return wrappedBatches; } + + public static List>> splitIntoBatchesWithID(Map objects) { + List>> wrappedBatches = new ArrayList<>(); + + int i = 0; + int j = 0; + + for (Entry entry : objects.entrySet()) { + T object = entry.getValue(); + if (wrappedBatches.size() <= j) { + wrappedBatches.add(new ArrayList<>()); + } + + wrappedBatches.get(j).add(new Container<>(object, entry.getKey())); + i++; + if (i % BATCH_SIZE == 0) { + j++; + } + } + return wrappedBatches; + } } diff --git a/Plan/src/main/java/com/djrapitops/plan/database/databases/MySQLDB.java b/Plan/src/main/java/com/djrapitops/plan/database/databases/MySQLDB.java index 21632a8f6..83fa8a665 100644 --- a/Plan/src/main/java/com/djrapitops/plan/database/databases/MySQLDB.java +++ b/Plan/src/main/java/com/djrapitops/plan/database/databases/MySQLDB.java @@ -35,7 +35,9 @@ public class MySQLDB extends SQLDB { try { Class.forName("com.mysql.jdbc.Driver"); - String url = "jdbc:mysql://" + config.getString("mysql.host") + ":" + config.getString("mysql.port") + "/" + config.getString("mysql.database"); + String url = "jdbc:mysql://" + config.getString("mysql.host") + ":" + config.getString("mysql.port") + "/" + + config.getString("mysql.database") + + "?rewriteBatchedStatements=true"; return DriverManager.getConnection(url, config.getString("mysql.user"), config.getString("mysql.password")); } catch (ClassNotFoundException | SQLException e) { diff --git a/Plan/src/main/java/com/djrapitops/plan/database/databases/SQLDB.java b/Plan/src/main/java/com/djrapitops/plan/database/databases/SQLDB.java index b78955a41..68ad0a490 100644 --- a/Plan/src/main/java/com/djrapitops/plan/database/databases/SQLDB.java +++ b/Plan/src/main/java/com/djrapitops/plan/database/databases/SQLDB.java @@ -424,19 +424,24 @@ public abstract class SQLDB extends Database { */ @Override public void saveMultipleUserData(Collection data) throws SQLException { - Benchmark.start("Database: Save multiple Userdata"); - checkConnection(); - if (data.isEmpty()) { + if (data == null || data.isEmpty()) { return; } + + Benchmark.start("Database: Save multiple Userdata"); + data.removeIf(Objects::isNull); + + checkConnection(); setStatus("Save userdata (multiple) for " + data.size()); usersTable.saveUserDataInformationBatch(data); // Transform to map - Map userDatas = data.stream().collect(Collectors.toMap(UserData::getUuid, Function.identity())); + Map userDatas = data.stream() + .collect(Collectors.toMap(UserData::getUuid, Function.identity())); + // Get UserIDs Map userIds = usersTable.getAllUserIds(); - // Empty dataset + // Create empty data sets Map> nicknames = new HashMap<>(); Map lastNicks = new HashMap<>(); Map> ips = new HashMap<>(); @@ -445,7 +450,8 @@ public abstract class SQLDB extends Database { Map> sessions = new HashMap<>(); Map> gmTimes = new HashMap<>(); Map> worldTimes = new HashMap<>(); - // Put to dataset + + // Put in data set List worldNames = data.stream() .map(UserData::getWorldTimes) .map(WorldTimes::getTimes) @@ -453,6 +459,7 @@ public abstract class SQLDB extends Database { .flatMap(Collection::stream) .distinct() .collect(Collectors.toList()); + for (Map.Entry entrySet : userDatas.entrySet()) { UUID uuid = entrySet.getKey(); UserData uData = entrySet.getValue(); @@ -472,6 +479,7 @@ public abstract class SQLDB extends Database { gmTimes.put(id, new HashMap<>(uData.getGmTimes().getTimes())); worldTimes.put(id, new HashMap<>(uData.getWorldTimes().getTimes())); } + // Save nicknamesTable.saveNickLists(nicknames, lastNicks); ipsTable.saveIPList(ips); diff --git a/Plan/src/main/java/com/djrapitops/plan/database/tables/CommandUseTable.java b/Plan/src/main/java/com/djrapitops/plan/database/tables/CommandUseTable.java index a9261f919..249625ae5 100644 --- a/Plan/src/main/java/com/djrapitops/plan/database/tables/CommandUseTable.java +++ b/Plan/src/main/java/com/djrapitops/plan/database/tables/CommandUseTable.java @@ -83,6 +83,7 @@ public class CommandUseTable extends Table { if (data.isEmpty()) { return; } + Benchmark.start("Database: Save Commanduse"); Map newData = new HashMap<>(data); Map saved = getCommandUse(); diff --git a/Plan/src/main/java/com/djrapitops/plan/database/tables/GMTimesTable.java b/Plan/src/main/java/com/djrapitops/plan/database/tables/GMTimesTable.java index c75481364..cdec6ff1f 100644 --- a/Plan/src/main/java/com/djrapitops/plan/database/tables/GMTimesTable.java +++ b/Plan/src/main/java/com/djrapitops/plan/database/tables/GMTimesTable.java @@ -2,6 +2,9 @@ package main.java.com.djrapitops.plan.database.tables; import com.djrapitops.plugin.utilities.Verify; import main.java.com.djrapitops.plan.Log; +import main.java.com.djrapitops.plan.data.time.GMTimes; +import main.java.com.djrapitops.plan.database.Container; +import main.java.com.djrapitops.plan.database.DBUtils; import main.java.com.djrapitops.plan.database.databases.SQLDB; import main.java.com.djrapitops.plan.utilities.Benchmark; @@ -93,12 +96,14 @@ public class GMTimesTable extends Table { statement.setInt(1, userId); set = statement.executeQuery(); HashMap times = new HashMap<>(); + while (set.next()) { times.put("SURVIVAL", set.getLong(columnSurvivalTime)); times.put("CREATIVE", set.getLong(columnCreativeTime)); times.put("ADVENTURE", set.getLong(columnAdventureTime)); times.put("SPECTATOR", set.getLong(columnSpectatorTime)); } + return times; } finally { close(set); @@ -113,18 +118,21 @@ public class GMTimesTable extends Table { try { statement = prepareStatement("SELECT * FROM " + tableName); set = statement.executeQuery(); + while (set.next()) { Map gmTimes = new HashMap<>(); int id = set.getInt(columnUserID); if (!userIds.contains(id)) { continue; } + gmTimes.put("SURVIVAL", set.getLong(columnSurvivalTime)); gmTimes.put("CREATIVE", set.getLong(columnCreativeTime)); gmTimes.put("ADVENTURE", set.getLong(columnAdventureTime)); gmTimes.put("SPECTATOR", set.getLong(columnSpectatorTime)); times.put(id, gmTimes); } + return times; } finally { close(set); @@ -141,8 +149,10 @@ public class GMTimesTable extends Table { if (Verify.isEmpty(gamemodeTimes)) { return; } + PreparedStatement statement = null; String[] gms = getGMKeyArray(); + int update; try { statement = prepareStatement( @@ -153,22 +163,17 @@ public class GMTimesTable extends Table { + columnSpectatorTime + "=? " + " WHERE (" + columnUserID + "=?)"); statement.setInt(5, userId); + for (int i = 0; i < gms.length; i++) { - try { - Long time = gamemodeTimes.get(gms[i]); - if (time != null) { - statement.setLong(i + 1, time); - } else { - statement.setLong(i + 1, 0); - } - } catch (NoSuchFieldError e) { - statement.setLong(i + 1, 0); - } + Long time = gamemodeTimes.get(gms[i]); + statement.setLong(i + 1, time != null ? time : 0); } + update = statement.executeUpdate(); } finally { close(statement); } + if (update == 0) { addNewGMTimesRow(userId, gamemodeTimes); } @@ -195,11 +200,52 @@ public class GMTimesTable extends Table { if (Verify.isEmpty(gamemodeTimes)) { return; } + Benchmark.start("Database: Save GMTimes"); - PreparedStatement statement = null; - String[] gms = getGMKeyArray(); Set savedIDs = getSavedIDs(); + + Map gmTimes = new HashMap<>(); + + for (Map.Entry> entrySet : gamemodeTimes.entrySet()) { + int userID = entrySet.getKey(); + + if (!savedIDs.contains(userID)) { + continue; + } + + Map gmTimesMap = entrySet.getValue(); + gmTimes.put(userID, new GMTimes(gmTimesMap)); + } + + List>> batches = DBUtils.splitIntoBatchesWithID(gmTimes); + + batches.parallelStream().forEach(batch -> { + try { + saveGMTimesBatch(batch); + } catch (SQLException e) { + Log.toLog("GMTimesTable.saveGMTimes", e); + } + }); + + gamemodeTimes.keySet().removeAll(savedIDs); + + addNewGMTimesRows(gamemodeTimes); + Benchmark.stop("Database: Save GMTimes"); + } + + private void saveGMTimesBatch(List> batch) throws SQLException { + if (batch.isEmpty()) { + return; + } + + int batchSize = batch.size(); + Log.debug("Preparing insertion of GM Times... Batch Size: " + batchSize); + + String[] gms = getGMKeyArray(); + Set savedIDs = getSavedIDs(); + + PreparedStatement statement = null; try { statement = prepareStatement( "UPDATE " + tableName + " SET " @@ -208,48 +254,72 @@ public class GMTimesTable extends Table { + columnAdventureTime + "=?, " + columnSpectatorTime + "=? " + " WHERE (" + columnUserID + "=?)"); - boolean commitRequired = false; - for (Map.Entry> entrySet : gamemodeTimes.entrySet()) { - Integer id = entrySet.getKey(); + + for (Container data : batch) { + int id = data.getId(); if (!savedIDs.contains(id)) { continue; } statement.setInt(5, id); + for (int i = 0; i < gms.length; i++) { - try { - Map times = entrySet.getValue(); - Long time = times.get(gms[i]); + Map times = data.getObject().getTimes(); + Long time = times.get(gms[i]); - statement.setLong(i + 1, time != null ? time : 0); - } catch (NoSuchFieldError e) { - statement.setLong(i + 1, 0); - } + statement.setLong(i + 1, time != null ? time : 0); } + statement.addBatch(); - commitRequired = true; } - if (commitRequired) { - statement.executeBatch(); - } - - gamemodeTimes.keySet().removeAll(savedIDs); + Log.debug("Executing GM Times batch: " + batchSize); + statement.executeBatch(); } finally { close(statement); } - - addNewGMTimesRows(gamemodeTimes); - Benchmark.stop("Database: Save GMTimes"); } private void addNewGMTimesRows(Map> gamemodeTimes) throws SQLException { if (Verify.isEmpty(gamemodeTimes)) { return; } - PreparedStatement statement = null; + + Benchmark.start("Database: Add GMTimes Rows"); + + Map gmTimes = new HashMap<>(); + + for (Map.Entry> entrySet : gamemodeTimes.entrySet()) { + int userID = entrySet.getKey(); + Map gmTimesMap = entrySet.getValue(); + gmTimes.put(userID, new GMTimes(gmTimesMap)); + } + + List>> batches = DBUtils.splitIntoBatchesWithID(gmTimes); + + batches.parallelStream().forEach(batch -> { + try { + addNewGMTimesBatch(batch); + } catch (SQLException e) { + e.printStackTrace(); + } + }); + + Benchmark.stop("Database: Add GMTimes Rows"); + } + + private void addNewGMTimesBatch(List> batch) throws SQLException { + if (batch.isEmpty()) { + return; + } + + int batchSize = batch.size(); + Log.debug("Preparing insertion of GM Times... Batch Size: " + batchSize); + String[] gms = getGMKeyArray(); + + PreparedStatement statement = null; try { statement = prepareStatement( "INSERT INTO " + tableName + " (" @@ -259,28 +329,22 @@ public class GMTimesTable extends Table { + columnAdventureTime + ", " + columnSpectatorTime + ") VALUES (?, ?, ?, ?, ?)"); - boolean commitRequired = false; - for (Map.Entry> entry : gamemodeTimes.entrySet()) { - Integer id = entry.getKey(); - statement.setInt(1, id); + for (Container data : batch) { + statement.setInt(1, data.getId()); + for (int i = 0; i < gms.length; i++) { - try { - Map times = entry.getValue(); - Long time = times.get(gms[i]); + Map times = data.getObject().getTimes(); + Long time = times.get(gms[i]); - statement.setLong(i + 2, time != null ? time : 0); - } catch (NoSuchFieldError e) { - statement.setLong(i + 2, 0); - } + statement.setLong(i + 2, time != null ? time : 0); } + statement.addBatch(); - commitRequired = true; } - if (commitRequired) { - statement.executeBatch(); - } + Log.debug("Executing GM Times batch: " + batchSize); + statement.executeBatch(); } finally { close(statement); } @@ -290,6 +354,7 @@ public class GMTimesTable extends Table { if (Verify.isEmpty(gamemodeTimes)) { return; } + PreparedStatement statement = null; String[] gms = getGMKeyArray(); try { @@ -302,14 +367,10 @@ public class GMTimesTable extends Table { + ") VALUES (?, ?, ?, ?, ?)"); statement.setInt(1, userId); - for (int i = 0; i < gms.length; i++) { - try { - Long time = gamemodeTimes.get(gms[i]); - statement.setLong(i + 2, time != null ? time : 0); - } catch (NoSuchFieldError e) { - statement.setLong(i + 2, 0); - } + for (int i = 0; i < gms.length; i++) { + Long time = gamemodeTimes.get(gms[i]); + statement.setLong(i + 2, time != null ? time : 0); } statement.execute(); diff --git a/Plan/src/main/java/com/djrapitops/plan/database/tables/IPsTable.java b/Plan/src/main/java/com/djrapitops/plan/database/tables/IPsTable.java index e07fd29d4..3e3cee558 100644 --- a/Plan/src/main/java/com/djrapitops/plan/database/tables/IPsTable.java +++ b/Plan/src/main/java/com/djrapitops/plan/database/tables/IPsTable.java @@ -90,6 +90,7 @@ public class IPsTable extends Table { Log.error("Host not found at getIPAddresses: " + ipAddressName); //Shouldn't ever happen } } + return ips; } finally { close(set); diff --git a/Plan/src/main/java/com/djrapitops/plan/database/tables/SessionsTable.java b/Plan/src/main/java/com/djrapitops/plan/database/tables/SessionsTable.java index b96052423..2eb135925 100644 --- a/Plan/src/main/java/com/djrapitops/plan/database/tables/SessionsTable.java +++ b/Plan/src/main/java/com/djrapitops/plan/database/tables/SessionsTable.java @@ -107,36 +107,38 @@ public class SessionsTable extends Table { if (sessions == null) { return; } + Benchmark.start("Database: Save Sessions"); sessions.removeAll(getSessionData(userId)); + if (sessions.isEmpty()) { + Benchmark.stop("Database: Save Sessions"); return; } - PreparedStatement statement = null; - try { + PreparedStatement statement = null; + + try { statement = prepareStatement("INSERT INTO " + tableName + " (" + columnUserID + ", " + columnSessionStart + ", " + columnSessionEnd + ") VALUES (?, ?, ?)"); - boolean commitRequired = false; for (SessionData session : sessions) { long end = session.getSessionEnd(); long start = session.getSessionStart(); if (end < start) { continue; } + statement.setInt(1, userId); statement.setLong(2, start); statement.setLong(3, end); statement.addBatch(); - commitRequired = true; - } - if (commitRequired) { - statement.executeBatch(); } + + statement.executeBatch(); } finally { close(statement); Benchmark.stop("Database: Save Sessions"); @@ -152,25 +154,31 @@ public class SessionsTable extends Table { if (ids == null || ids.isEmpty()) { return new HashMap<>(); } + Benchmark.start("Database: Get Sessions multiple"); PreparedStatement statement = null; ResultSet set = null; + try { Map> sessions = new HashMap<>(); statement = prepareStatement("SELECT * FROM " + tableName); set = statement.executeQuery(); + for (Integer id : ids) { sessions.put(id, new ArrayList<>()); } + while (set.next()) { Integer id = set.getInt(columnUserID); if (!ids.contains(id)) { continue; } - sessions.get(id).add(new SessionData(set.getLong(columnSessionStart), set.getLong(columnSessionEnd))); + + long sessionStart = set.getLong(columnSessionStart); + long sessionEnd = set.getLong(columnSessionEnd); + + sessions.get(id).add(new SessionData(sessionStart, sessionEnd)); } - set.close(); - statement.close(); return sessions; } finally { @@ -195,8 +203,8 @@ public class SessionsTable extends Table { for (Map.Entry> entrySet : sessions.entrySet()) { Integer id = entrySet.getKey(); List sessionList = entrySet.getValue(); - List s = saved.get(id); + if (s != null) { sessionList.removeAll(s); } @@ -207,11 +215,17 @@ public class SessionsTable extends Table { saved.put(id, sessionList); } + List>> batches = splitIntoBatches(sessions); - for (List> batch : batches) { - saveSessionBatch(batch); - } + batches.parallelStream().forEach(batch -> { + try { + saveSessionBatch(batch); + } catch (SQLException e) { + e.printStackTrace(); + } + }); + Benchmark.stop("Database: Save Sessions multiple"); } @@ -219,6 +233,10 @@ public class SessionsTable extends Table { if (batch.isEmpty()) { return; } + + int batchSize = batch.size(); + Log.debug("Preparing insertion of sessions... Batch Size: " + batchSize); + PreparedStatement statement = null; try { statement = prepareStatement("INSERT INTO " + tableName + " (" @@ -227,25 +245,21 @@ public class SessionsTable extends Table { + columnSessionEnd + ") VALUES (?, ?, ?)"); - boolean commitRequired = false; - int i = 0; for (Container data : batch) { SessionData session = data.getObject(); int id = data.getId(); if (!session.isValid()) { continue; } + statement.setInt(1, id); statement.setLong(2, session.getSessionStart()); statement.setLong(3, session.getSessionEnd()); statement.addBatch(); - commitRequired = true; - i++; - } - if (commitRequired) { - Log.debug("Executing session batch: " + i); - statement.executeBatch(); } + + Log.debug("Executing session batch: " + batchSize); + statement.executeBatch(); } finally { close(statement); } diff --git a/Plan/src/main/java/com/djrapitops/plan/database/tables/TPSTable.java b/Plan/src/main/java/com/djrapitops/plan/database/tables/TPSTable.java index 55921e575..f2b9b86a8 100644 --- a/Plan/src/main/java/com/djrapitops/plan/database/tables/TPSTable.java +++ b/Plan/src/main/java/com/djrapitops/plan/database/tables/TPSTable.java @@ -119,12 +119,24 @@ public class TPSTable extends Table { */ public void saveTPSData(List data) throws SQLException { List> batches = DBUtils.splitIntoBatches(data); - for (List batch : batches) { - saveTPSBatch(batch); - } + batches.parallelStream() + .forEach(batch -> { + try { + saveTPSBatch(batch); + } catch (SQLException e) { + Log.toLog("UsersTable.saveUserDataInformationBatch", e); + } + }); } private void saveTPSBatch(List batch) throws SQLException { + if (batch.isEmpty()) { + return; + } + + int batchSize = batch.size(); + Log.debug("Preparing insertion of TPS... Batch Size: " + batchSize); + PreparedStatement statement = null; try { statement = prepareStatement("INSERT INTO " + tableName + " (" @@ -137,8 +149,6 @@ public class TPSTable extends Table { + columnChunksLoaded + ") VALUES (?, ?, ?, ?, ?, ?, ?)"); - boolean commitRequired = false; - int i = 0; for (TPS tps : batch) { statement.setLong(1, tps.getDate()); statement.setDouble(2, tps.getTps()); @@ -148,13 +158,10 @@ public class TPSTable extends Table { statement.setDouble(6, tps.getEntityCount()); statement.setDouble(7, tps.getChunksLoaded()); statement.addBatch(); - commitRequired = true; - i++; - } - if (commitRequired) { - Log.debug("Executing tps batch: " + i); - statement.executeBatch(); } + + Log.debug("Executing tps batch: " + batchSize); + statement.executeBatch(); } finally { close(statement); } diff --git a/Plan/src/main/java/com/djrapitops/plan/database/tables/UsersTable.java b/Plan/src/main/java/com/djrapitops/plan/database/tables/UsersTable.java index 5d31f72b6..2d20f4db3 100644 --- a/Plan/src/main/java/com/djrapitops/plan/database/tables/UsersTable.java +++ b/Plan/src/main/java/com/djrapitops/plan/database/tables/UsersTable.java @@ -680,10 +680,18 @@ public class UsersTable extends Table { try { List newUserdata = updateExistingUserData(data); Benchmark.start("Database: Insert new UserInfo multiple"); + List> batches = DBUtils.splitIntoBatches(newUserdata); - for (List batch : batches) { - insertNewUserData(batch); - } + + batches.parallelStream() + .forEach(batch -> { + try { + insertNewUserData(batch); + } catch (SQLException e) { + Log.toLog("UsersTable.saveUserDataInformationBatch", e); + } + }); + Benchmark.stop("Database: Insert new UserInfo multiple"); } finally { Benchmark.stop("Database: Save UserInfo multiple"); @@ -691,18 +699,27 @@ public class UsersTable extends Table { } private void insertNewUserData(Collection data) throws SQLException { + if (data.isEmpty()) { + return; + } + + int batchSize = data.size(); + Log.debug("Preparing insertion of new users... Batch Size: " + batchSize); + PreparedStatement statement = null; try { statement = prepareStatement(getInsertStatement()); - boolean commitRequired = false; - int i = 0; + for (UserData uData : data) { UUID uuid = uData.getUuid(); + statement.setString(1, uuid.toString()); statement.setString(2, uData.getGeolocation()); + GMTimes gmTimes = uData.getGmTimes(); statement.setString(3, gmTimes.getState()); statement.setLong(4, gmTimes.getLastStateChange()); + statement.setLong(5, uData.getPlayTime()); statement.setInt(6, uData.getLoginTimes()); statement.setLong(7, uData.getLastPlayed()); @@ -713,17 +730,16 @@ public class UsersTable extends Table { statement.setBoolean(12, uData.isBanned()); statement.setString(13, uData.getName()); statement.setLong(14, uData.getRegistered()); + WorldTimes worldTimes = uData.getWorldTimes(); statement.setString(15, worldTimes.getState()); statement.setLong(16, worldTimes.getLastStateChange()); + statement.addBatch(); - commitRequired = true; - i++; - } - if (commitRequired) { - Log.debug("Executing session batch: " + i); - statement.executeBatch(); } + + Log.debug("Executing users batch: " + batchSize); + statement.executeBatch(); } finally { close(statement); }