mirror/Plan
2
0
Fork 0
mirror of https://github.com/plan-player-analytics/Plan.git synced 2025-01-06 15:44:49 +08:00
Code Issues Packages Projects Releases Wiki Activity

Server page splitting for #531 + b3

Browse Source
This commit is contained in:
Rsl1122 2018-02-28 15:47:37 +02:00
parent 385e161d37
commit eba17eb127
9 changed files with 222 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
Plan/src/main/java/com/djrapitops/plan/system/database/databases/sql/SQLDB.java
View File

@ -134,7 +134,7 @@ public abstract class SQLDB extends Database {
if (newDatabase) { if (newDatabase) {
Log.info("New Database created."); Log.info("New Database created.");
versionTable.setVersion(13); versionTable.setVersion(14);
} }
int version = versionTable.getVersion(); int version = versionTable.getVersion();
@ -165,6 +165,10 @@ public abstract class SQLDB extends Database {
geoInfoTable.alterTableV13(); geoInfoTable.alterTableV13();
versionTable.setVersion(13); versionTable.setVersion(13);
} }
if (version < 14) {
transferTable.alterTableV14();
versionTable.setVersion(14);
}
} catch (SQLException e) { } catch (SQLException e) {
throw new DBInitException("Failed to set-up Database", e); throw new DBInitException("Failed to set-up Database", e);
} }

20
Plan/src/main/java/com/djrapitops/plan/system/database/databases/sql/statements/Column.java Normal file
View File

@ -0,0 +1,20 @@
/*
* Licence is provided in the jar as license.yml also here:
* https://github.com/Rsl1122/Plan-PlayerAnalytics/blob/master/Plan/src/main/resources/license.yml
*/
package com.djrapitops.plan.system.database.databases.sql.statements;
/**
* Interface for SQL column enum compatibility.
*
* @author Rsl1122
*/
public interface Column {
default String get() {
return toString();
}
String toString();
}

8
Plan/src/main/java/com/djrapitops/plan/system/database/databases/sql/statements/TableSqlParser.java
View File

@ -32,6 +32,10 @@ public class TableSqlParser extends SqlParser {
return new TableSqlParser("").column(column, type); return new TableSqlParser("").column(column, type);
} }
public TableSqlParser column(Column column, String type) {
return column(column.get(), type);
}
public TableSqlParser column(String column, String type) { public TableSqlParser column(String column, String type) {
if (columns > 0) { if (columns > 0) {
append(", "); append(", ");
@ -43,6 +47,10 @@ public class TableSqlParser extends SqlParser {
return this; return this;
} }
public TableSqlParser foreignKey(Column column, String refrencedTable, Column referencedColumn) {
return foreignKey(column.get(), refrencedTable, referencedColumn.get());
}
public TableSqlParser foreignKey(String column, String refrencedTable, String referencedColumn) { public TableSqlParser foreignKey(String column, String refrencedTable, String referencedColumn) {
if (columns > 0) { if (columns > 0) {
append(", "); append(", ");

34
Plan/src/main/java/com/djrapitops/plan/system/database/databases/sql/tables/ServerTable.java
View File

@ -35,6 +35,11 @@ import java.util.*;
*/ */
public class ServerTable extends Table { public class ServerTable extends Table {
@Deprecated
public String getColumnID() {
return Col.SERVER_ID.get();
}
public final String statementSelectServerID; public final String statementSelectServerID;
public final String statementSelectServerNameID; public final String statementSelectServerNameID;
private static final String columnServerID = "id"; private static final String columnServerID = "id";
@ -308,12 +313,33 @@ public class ServerTable extends Table {
}); });
} }
public String getColumnID() { @Deprecated
return columnServerID; public String getColumnUUID() {
return Col.SERVER_UUID.get();
} }
public String getColumnUUID() { public enum Col implements Column {
return columnServerUUID; SERVER_ID("id"),
SERVER_UUID("uuid"),
NAME("name"),
WEBSERVER_ADDRESS("web_address"),
INSTALLED("is_installed"),
MAX_PLAYERS("max_players");
private final String column;
Col(String column) {
this.column = column;
}
public String get() {
return toString();
}
@Override
public String toString() {
return column;
}
} }
public void insertAllServers(List<Server> allServer) throws SQLException { public void insertAllServers(List<Server> allServer) throws SQLException {

182
Plan/src/main/java/com/djrapitops/plan/system/database/databases/sql/tables/TransferTable.java
View File

@ -8,6 +8,7 @@ import com.djrapitops.plan.api.exceptions.database.DBInitException;
import com.djrapitops.plan.system.database.databases.sql.SQLDB; import com.djrapitops.plan.system.database.databases.sql.SQLDB;
import com.djrapitops.plan.system.database.databases.sql.processing.ExecStatement; import com.djrapitops.plan.system.database.databases.sql.processing.ExecStatement;
import com.djrapitops.plan.system.database.databases.sql.processing.QueryStatement; import com.djrapitops.plan.system.database.databases.sql.processing.QueryStatement;
import com.djrapitops.plan.system.database.databases.sql.statements.Column;
import com.djrapitops.plan.system.database.databases.sql.statements.Sql; import com.djrapitops.plan.system.database.databases.sql.statements.Sql;
import com.djrapitops.plan.system.database.databases.sql.statements.TableSqlParser; import com.djrapitops.plan.system.database.databases.sql.statements.TableSqlParser;
import com.djrapitops.plan.system.info.request.CacheAnalysisPageRequest; import com.djrapitops.plan.system.info.request.CacheAnalysisPageRequest;
@ -15,16 +16,14 @@ import com.djrapitops.plan.system.info.request.CacheInspectPageRequest;
import com.djrapitops.plan.system.info.request.CacheInspectPluginsTabRequest; import com.djrapitops.plan.system.info.request.CacheInspectPluginsTabRequest;
import com.djrapitops.plan.system.info.request.CacheNetworkPageContentRequest; import com.djrapitops.plan.system.info.request.CacheNetworkPageContentRequest;
import com.djrapitops.plan.system.info.server.ServerInfo; import com.djrapitops.plan.system.info.server.ServerInfo;
import com.djrapitops.plan.utilities.Base64Util;
import com.djrapitops.plan.utilities.MiscUtils; import com.djrapitops.plan.utilities.MiscUtils;
import com.djrapitops.plugin.api.TimeAmount; import com.djrapitops.plugin.api.TimeAmount;
import java.sql.PreparedStatement; import java.sql.PreparedStatement;
import java.sql.ResultSet; import java.sql.ResultSet;
import java.sql.SQLException; import java.sql.SQLException;
import java.util.HashMap; import java.util.*;
import java.util.Map;
import java.util.Optional;
import java.util.UUID;
/** /**
* Table that represents plan_transfer in SQL database. * Table that represents plan_transfer in SQL database.
@ -33,54 +32,65 @@ import java.util.UUID;
*/ */
public class TransferTable extends Table { public class TransferTable extends Table {
private static final String columnSenderID = "sender_server_id"; private final String insertStatementNoParts;
private static final String columnExpiry = "expiry_date";
private static final String columnInfoType = "type";
private static final String columnContent = "content_64";
private static final String columnExtraVariables = "extra_variables";
private final ServerTable serverTable; private final ServerTable serverTable;
private final String insertStatementParts;
private final String insertStatement;
private final String selectStatement; private final String selectStatement;
public TransferTable(SQLDB db) { public TransferTable(SQLDB db) {
super("plan_transfer", db); super("plan_transfer", db);
serverTable = db.getServerTable(); serverTable = db.getServerTable();
insertStatement = "INSERT INTO " + tableName + " (" + insertStatementParts = "REPLACE INTO " + tableName + " (" +
columnSenderID + ", " + Col.SENDER_ID + ", " +
columnExpiry + ", " + Col.EXPIRY + ", " +
columnInfoType + ", " + Col.INFO_TYPE + ", " +
columnExtraVariables + ", " + Col.EXTRA_VARIABLES + ", " +
columnContent + Col.CONTENT + ", " +
Col.PART +
") VALUES (" +
serverTable.statementSelectServerID + ", " +
"?, ?, ?, ?, ?)";
insertStatementNoParts = "REPLACE INTO " + tableName + " (" +
Col.SENDER_ID + ", " +
Col.EXPIRY + ", " +
Col.INFO_TYPE + ", " +
Col.EXTRA_VARIABLES + ", " +
Col.CONTENT +
") VALUES (" + ") VALUES (" +
serverTable.statementSelectServerID + ", " + serverTable.statementSelectServerID + ", " +
"?, ?, ?, ?)"; "?, ?, ?, ?)";
selectStatement = "SELECT * FROM " + tableName + selectStatement = "SELECT * FROM " + tableName +
" WHERE " + columnInfoType + "= ?" + " WHERE " + Col.INFO_TYPE + "= ?" +
" AND " + columnExpiry + "> ?" + " AND " + Col.EXPIRY + "> ?" +
" ORDER BY " + columnExpiry + " DESC"; " ORDER BY " + Col.EXPIRY + " DESC, "
+ Col.PART + " ASC";
} }
@Override @Override
public void createTable() throws DBInitException { public void createTable() throws DBInitException {
createTable(TableSqlParser.createTable(tableName) createTable(TableSqlParser.createTable(tableName)
.column(columnSenderID, Sql.INT).notNull() .column(Col.SENDER_ID, Sql.INT).notNull()
.column(columnExpiry, Sql.LONG).notNull().defaultValue("0") .column(Col.EXPIRY, Sql.LONG).notNull().defaultValue("0")
.column(columnInfoType, Sql.varchar(100)).notNull() .column(Col.INFO_TYPE, Sql.varchar(100)).notNull()
.column(columnExtraVariables, Sql.varchar(255)).defaultValue("''") .column(Col.EXTRA_VARIABLES, Sql.varchar(255)).defaultValue("''")
.column(columnContent, usingMySQL ? "MEDIUMTEXT" : Sql.varchar(1)) // SQLite does not enforce varchar limits. .column(Col.CONTENT, usingMySQL ? "MEDIUMTEXT" : Sql.varchar(1)) // SQLite does not enforce varchar limits.
.foreignKey(columnSenderID, serverTable.toString(), serverTable.getColumnID()) .column(Col.PART, Sql.LONG).notNull().defaultValue("0")
.foreignKey(Col.SENDER_ID, serverTable.toString(), ServerTable.Col.SERVER_ID)
.toString() .toString()
); );
} }
public void alterTableV14() {
addColumns(Col.PART + " bigint NOT NULL DEFAULT 0");
}
public void clean() throws SQLException { public void clean() throws SQLException {
String sql = "DELETE FROM " + tableName + String sql = "DELETE FROM " + tableName +
" WHERE " + columnExpiry + " < ?" + " WHERE " + Col.EXPIRY + " < ?" +
" AND " + columnInfoType + " != ?"; " AND " + Col.INFO_TYPE + " != ?";
execute(new ExecStatement(sql) { execute(new ExecStatement(sql) {
@Override @Override
@ -90,8 +100,8 @@ public class TransferTable extends Table {
} }
}); });
sql = "DELETE FROM " + tableName + sql = "DELETE FROM " + tableName +
" WHERE " + columnSenderID + " = " + serverTable.statementSelectServerID + " WHERE " + Col.SENDER_ID + " = " + serverTable.statementSelectServerID +
" AND " + columnInfoType + " = ?"; " AND " + Col.INFO_TYPE + " = ?";
execute(new ExecStatement(sql) { execute(new ExecStatement(sql) {
@Override @Override
@ -103,7 +113,7 @@ public class TransferTable extends Table {
} }
public void storePlayerHtml(UUID player, String encodedHtml) throws SQLException { public void storePlayerHtml(UUID player, String encodedHtml) throws SQLException {
execute(new ExecStatement(insertStatement) { execute(new ExecStatement(insertStatementNoParts) {
@Override @Override
public void prepare(PreparedStatement statement) throws SQLException { public void prepare(PreparedStatement statement) throws SQLException {
statement.setString(1, ServerInfo.getServerUUID().toString()); statement.setString(1, ServerInfo.getServerUUID().toString());
@ -116,20 +126,28 @@ public class TransferTable extends Table {
} }
public void storeServerHtml(UUID serverUUID, String encodedHtml) throws SQLException { public void storeServerHtml(UUID serverUUID, String encodedHtml) throws SQLException {
execute(new ExecStatement(insertStatement) { List<String> split = Base64Util.split(encodedHtml, 500000L);
@Override
public void prepare(PreparedStatement statement) throws SQLException { int i = 0;
statement.setString(1, ServerInfo.getServerUUID().toString()); for (String part : split) {
statement.setLong(2, MiscUtils.getTime() + TimeAmount.MINUTE.ms()); final int partNumber = i;
statement.setString(3, CacheAnalysisPageRequest.class.getSimpleName().toLowerCase()); execute(new ExecStatement(insertStatementParts) {
statement.setString(4, serverUUID.toString()); @Override
statement.setString(5, encodedHtml); public void prepare(PreparedStatement statement) throws SQLException {
} statement.setString(1, ServerInfo.getServerUUID().toString());
}); statement.setLong(2, MiscUtils.getTime() + TimeAmount.MINUTE.ms());
statement.setString(3, CacheAnalysisPageRequest.class.getSimpleName().toLowerCase());
statement.setString(4, serverUUID.toString());
statement.setString(5, part);
statement.setInt(6, partNumber);
}
});
i++;
}
} }
public void storeNetworkPageContent(UUID serverUUID, String encodedHtml) throws SQLException { public void storeNetworkPageContent(UUID serverUUID, String encodedHtml) throws SQLException {
execute(new ExecStatement(insertStatement) { execute(new ExecStatement(insertStatementNoParts) {
@Override @Override
public void prepare(PreparedStatement statement) throws SQLException { public void prepare(PreparedStatement statement) throws SQLException {
statement.setString(1, ServerInfo.getServerUUID().toString()); statement.setString(1, ServerInfo.getServerUUID().toString());
@ -153,18 +171,29 @@ public class TransferTable extends Table {
public Map<UUID, String> processResults(ResultSet set) throws SQLException { public Map<UUID, String> processResults(ResultSet set) throws SQLException {
Map<UUID, String> htmlPerUUID = new HashMap<>(); Map<UUID, String> htmlPerUUID = new HashMap<>();
while (set.next()) { while (set.next()) {
String uuidString = set.getString(columnExtraVariables); String uuidString = set.getString(Col.EXTRA_VARIABLES.get());
UUID uuid = UUID.fromString(uuidString); UUID uuid = UUID.fromString(uuidString);
if (!htmlPerUUID.containsKey(uuid)) { htmlPerUUID.put(uuid, htmlPerUUID.getOrDefault(uuid, "") + set.getString(Col.CONTENT.get()));
htmlPerUUID.put(uuid, set.getString(columnContent));
}
} }
return htmlPerUUID; return htmlPerUUID;
} }
}); });
} }
public void storePlayerPluginsTab(UUID player, String encodedHtml) throws SQLException {
execute(new ExecStatement(insertStatementNoParts) {
@Override
public void prepare(PreparedStatement statement) throws SQLException {
statement.setString(1, ServerInfo.getServerUUID().toString());
statement.setLong(2, MiscUtils.getTime() + TimeAmount.MINUTE.ms());
statement.setString(3, CacheInspectPluginsTabRequest.class.getSimpleName().toLowerCase());
statement.setString(4, player.toString());
statement.setString(5, encodedHtml);
}
});
}
public Map<UUID, String> getPlayerHtml() throws SQLException { public Map<UUID, String> getPlayerHtml() throws SQLException {
return getHtmlPerUUIDForCacheRequest(CacheInspectPageRequest.class); return getHtmlPerUUIDForCacheRequest(CacheInspectPageRequest.class);
} }
@ -177,30 +206,17 @@ public class TransferTable extends Table {
return getHtmlPerUUIDForCacheRequest(CacheAnalysisPageRequest.class); return getHtmlPerUUIDForCacheRequest(CacheAnalysisPageRequest.class);
} }
public void storePlayerPluginsTab(UUID player, String encodedHtml) throws SQLException {
execute(new ExecStatement(insertStatement) {
@Override
public void prepare(PreparedStatement statement) throws SQLException {
statement.setString(1, ServerInfo.getServerUUID().toString());
statement.setLong(2, MiscUtils.getTime() + TimeAmount.MINUTE.ms());
statement.setString(3, CacheInspectPluginsTabRequest.class.getSimpleName().toLowerCase());
statement.setString(4, player.toString());
statement.setString(5, encodedHtml);
}
});
}
public Map<UUID, String> getPlayerPluginsTabs(UUID playerUUID) throws SQLException { public Map<UUID, String> getPlayerPluginsTabs(UUID playerUUID) throws SQLException {
String serverIDColumn = serverTable + "." + serverTable.getColumnID(); String serverIDColumn = serverTable + "." + serverTable.getColumnID();
String serverUUIDColumn = serverTable + "." + serverTable.getColumnUUID() + " as s_uuid"; String serverUUIDColumn = serverTable + "." + serverTable.getColumnUUID() + " as s_uuid";
String sql = "SELECT " + String sql = "SELECT " +
columnContent + ", " + Col.CONTENT + ", " +
serverUUIDColumn + serverUUIDColumn +
" FROM " + tableName + " FROM " + tableName +
" INNER JOIN " + serverTable + " on " + serverIDColumn + "=" + columnSenderID + " INNER JOIN " + serverTable + " on " + serverIDColumn + "=" + Col.SENDER_ID +
" WHERE " + columnInfoType + "= ?" + " WHERE " + Col.INFO_TYPE + "= ?" +
" AND " + columnExpiry + "> ?" + " AND " + Col.EXPIRY + "> ?" +
" AND " + columnExtraVariables + "=?"; " AND " + Col.EXTRA_VARIABLES + "=?";
return query(new QueryStatement<Map<UUID, String>>(sql, 250) { return query(new QueryStatement<Map<UUID, String>>(sql, 250) {
@Override @Override
@ -215,7 +231,7 @@ public class TransferTable extends Table {
Map<UUID, String> htmlPerUUID = new HashMap<>(); Map<UUID, String> htmlPerUUID = new HashMap<>();
while (set.next()) { while (set.next()) {
UUID serverUUID = UUID.fromString(set.getString("s_uuid")); UUID serverUUID = UUID.fromString(set.getString("s_uuid"));
String html64 = set.getString(columnContent); String html64 = set.getString(Col.CONTENT.get());
htmlPerUUID.put(serverUUID, html64); htmlPerUUID.put(serverUUID, html64);
} }
@ -231,9 +247,9 @@ public class TransferTable extends Table {
String sql = "SELECT " + String sql = "SELECT " +
serverUUIDColumn + serverUUIDColumn +
" FROM " + tableName + " FROM " + tableName +
" INNER JOIN " + serverTable + " on " + serverIDColumn + "=" + columnSenderID + " INNER JOIN " + serverTable + " on " + serverIDColumn + "=" + Col.SENDER_ID +
" WHERE " + columnExtraVariables + "=?" + " WHERE " + Col.EXTRA_VARIABLES + "=?" +
" ORDER BY " + columnExpiry + " LIMIT 1"; " ORDER BY " + Col.EXPIRY + " LIMIT 1";
return query(new QueryStatement<Optional<UUID>>(sql, 1) { return query(new QueryStatement<Optional<UUID>>(sql, 1) {
@Override @Override
@ -253,7 +269,7 @@ public class TransferTable extends Table {
@Deprecated @Deprecated
public void storePlayerOnlineOnThisServer(UUID playerUUID) throws SQLException { public void storePlayerOnlineOnThisServer(UUID playerUUID) throws SQLException {
execute(new ExecStatement(insertStatement) { execute(new ExecStatement(insertStatementNoParts) {
@Override @Override
public void prepare(PreparedStatement statement) throws SQLException { public void prepare(PreparedStatement statement) throws SQLException {
statement.setString(1, ServerInfo.getServerUUID().toString()); statement.setString(1, ServerInfo.getServerUUID().toString());
@ -266,7 +282,7 @@ public class TransferTable extends Table {
} }
public void storeConfigSettings(String encodedSettingString) throws SQLException { public void storeConfigSettings(String encodedSettingString) throws SQLException {
execute(new ExecStatement(insertStatement) { execute(new ExecStatement(insertStatementNoParts) {
@Override @Override
public void prepare(PreparedStatement statement) throws SQLException { public void prepare(PreparedStatement statement) throws SQLException {
statement.setString(1, ServerInfo.getServerUUID().toString()); statement.setString(1, ServerInfo.getServerUUID().toString());
@ -289,10 +305,34 @@ public class TransferTable extends Table {
@Override @Override
public Optional<String> processResults(ResultSet set) throws SQLException { public Optional<String> processResults(ResultSet set) throws SQLException {
if (set.next()) { if (set.next()) {
return Optional.ofNullable(set.getString(columnContent)); return Optional.ofNullable(set.getString(Col.CONTENT.get()));
} }
return Optional.empty(); return Optional.empty();
} }
}); });
} }
public enum Col implements Column {
SENDER_ID("sender_server_id"),
EXPIRY("expiry_date"),
INFO_TYPE("type"),
CONTENT("content_64"),
EXTRA_VARIABLES("extra_variables"),
PART("part");
private final String column;
Col(String column) {
this.column = column;
}
public String get() {
return toString();
}
@Override
public String toString() {
return column;
}
}
} }

23
Plan/src/main/java/com/djrapitops/plan/utilities/Base64Util.java
View File

@ -4,7 +4,9 @@
*/ */
package com.djrapitops.plan.utilities; package com.djrapitops.plan.utilities;
import java.util.ArrayList;
import java.util.Base64; import java.util.Base64;
import java.util.List;
/** /**
* Utility for performing Base64 operations. * Utility for performing Base64 operations.
@ -29,4 +31,25 @@ public class Base64Util {
return new String(decoded); return new String(decoded);
} }
public static List<String> split(String encoded, long partLength) {
List<String> split = new ArrayList<>();
StringBuilder builder = new StringBuilder();
long length = 0;
for (char c : encoded.toCharArray()) {
builder.append(c);
length++;
if (length >= partLength) {
split.add(builder.toString());
builder = new StringBuilder();
length = 0;
}
}
// Add the last part even if it isn't full length.
if (length != 0) {
split.add(builder.toString());
}
return split;
}
} }

2
Plan/src/main/resources/bungee.yml
View File

@ -1,4 +1,4 @@
name: Plan name: Plan
author: Rsl1122 author: Rsl1122
main: com.djrapitops.plan.PlanBungee main: com.djrapitops.plan.PlanBungee
version: 4.1.6-b2 version: 4.1.6-b3

2
Plan/src/main/resources/plugin.yml
View File

@ -1,7 +1,7 @@
name: Plan name: Plan
author: Rsl1122 author: Rsl1122
main: com.djrapitops.plan.Plan main: com.djrapitops.plan.Plan
version: 4.1.6-b2 version: 4.1.6-b3
softdepend: softdepend:
- EssentialsX - EssentialsX
- Towny - Towny

23
Plan/src/test/java/com/djrapitops/plan/system/database/databases/SQLiteTest.java
View File

@ -896,4 +896,27 @@ public class SQLiteTest {
assertTrue(configSettings.isPresent()); assertTrue(configSettings.isPresent());
assertEquals(testString, Base64Util.decode(configSettings.get())); assertEquals(testString, Base64Util.decode(configSettings.get()));
} }
@Test
public void testTransferSplitAndRearrange() throws SQLException {
StringBuilder testData = new StringBuilder();
for (int i = 0; i < 900000; i++) {
testData.append("a");
}
String testString = Base64Util.encode(testData.toString());
int length = testString.length();
System.out.println("Test String Length: " + length);
assertTrue(length > 500000);
System.out.println("Test Parts: " + (int) Math.ceil(length / 500000.0));
TransferTable transferTable = db.getTransferTable();
transferTable.storeServerHtml(TestConstants.SERVER_UUID, testString);
String result = transferTable.getServerHtml().get(TestConstants.SERVER_UUID);
assertNotNull(result);
assertEquals(testString, result);
}
} }