mirror of
https://github.com/plan-player-analytics/Plan.git
synced 2025-01-06 15:44:49 +08:00
Server page splitting for #531 + b3
This commit is contained in:
parent
385e161d37
commit
eba17eb127
Plan/src
main
java/com/djrapitops/plan
system/database/databases/sql
utilities
resources
test/java/com/djrapitops/plan/system/database/databases
@ -134,7 +134,7 @@ public abstract class SQLDB extends Database {
|
||||
|
||||
if (newDatabase) {
|
||||
Log.info("New Database created.");
|
||||
versionTable.setVersion(13);
|
||||
versionTable.setVersion(14);
|
||||
}
|
||||
|
||||
int version = versionTable.getVersion();
|
||||
@ -165,6 +165,10 @@ public abstract class SQLDB extends Database {
|
||||
geoInfoTable.alterTableV13();
|
||||
versionTable.setVersion(13);
|
||||
}
|
||||
if (version < 14) {
|
||||
transferTable.alterTableV14();
|
||||
versionTable.setVersion(14);
|
||||
}
|
||||
} catch (SQLException e) {
|
||||
throw new DBInitException("Failed to set-up Database", e);
|
||||
}
|
||||
|
@ -0,0 +1,20 @@
|
||||
/*
|
||||
* Licence is provided in the jar as license.yml also here:
|
||||
* https://github.com/Rsl1122/Plan-PlayerAnalytics/blob/master/Plan/src/main/resources/license.yml
|
||||
*/
|
||||
package com.djrapitops.plan.system.database.databases.sql.statements;
|
||||
|
||||
/**
|
||||
* Interface for SQL column enum compatibility.
|
||||
*
|
||||
* @author Rsl1122
|
||||
*/
|
||||
public interface Column {
|
||||
|
||||
default String get() {
|
||||
return toString();
|
||||
}
|
||||
|
||||
String toString();
|
||||
|
||||
}
|
@ -32,6 +32,10 @@ public class TableSqlParser extends SqlParser {
|
||||
return new TableSqlParser("").column(column, type);
|
||||
}
|
||||
|
||||
public TableSqlParser column(Column column, String type) {
|
||||
return column(column.get(), type);
|
||||
}
|
||||
|
||||
public TableSqlParser column(String column, String type) {
|
||||
if (columns > 0) {
|
||||
append(", ");
|
||||
@ -43,6 +47,10 @@ public class TableSqlParser extends SqlParser {
|
||||
return this;
|
||||
}
|
||||
|
||||
public TableSqlParser foreignKey(Column column, String refrencedTable, Column referencedColumn) {
|
||||
return foreignKey(column.get(), refrencedTable, referencedColumn.get());
|
||||
}
|
||||
|
||||
public TableSqlParser foreignKey(String column, String refrencedTable, String referencedColumn) {
|
||||
if (columns > 0) {
|
||||
append(", ");
|
||||
|
@ -35,6 +35,11 @@ import java.util.*;
|
||||
*/
|
||||
public class ServerTable extends Table {
|
||||
|
||||
@Deprecated
|
||||
public String getColumnID() {
|
||||
return Col.SERVER_ID.get();
|
||||
}
|
||||
|
||||
public final String statementSelectServerID;
|
||||
public final String statementSelectServerNameID;
|
||||
private static final String columnServerID = "id";
|
||||
@ -308,12 +313,33 @@ public class ServerTable extends Table {
|
||||
});
|
||||
}
|
||||
|
||||
public String getColumnID() {
|
||||
return columnServerID;
|
||||
@Deprecated
|
||||
public String getColumnUUID() {
|
||||
return Col.SERVER_UUID.get();
|
||||
}
|
||||
|
||||
public String getColumnUUID() {
|
||||
return columnServerUUID;
|
||||
public enum Col implements Column {
|
||||
SERVER_ID("id"),
|
||||
SERVER_UUID("uuid"),
|
||||
NAME("name"),
|
||||
WEBSERVER_ADDRESS("web_address"),
|
||||
INSTALLED("is_installed"),
|
||||
MAX_PLAYERS("max_players");
|
||||
|
||||
private final String column;
|
||||
|
||||
Col(String column) {
|
||||
this.column = column;
|
||||
}
|
||||
|
||||
public String get() {
|
||||
return toString();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return column;
|
||||
}
|
||||
}
|
||||
|
||||
public void insertAllServers(List<Server> allServer) throws SQLException {
|
||||
|
@ -8,6 +8,7 @@ import com.djrapitops.plan.api.exceptions.database.DBInitException;
|
||||
import com.djrapitops.plan.system.database.databases.sql.SQLDB;
|
||||
import com.djrapitops.plan.system.database.databases.sql.processing.ExecStatement;
|
||||
import com.djrapitops.plan.system.database.databases.sql.processing.QueryStatement;
|
||||
import com.djrapitops.plan.system.database.databases.sql.statements.Column;
|
||||
import com.djrapitops.plan.system.database.databases.sql.statements.Sql;
|
||||
import com.djrapitops.plan.system.database.databases.sql.statements.TableSqlParser;
|
||||
import com.djrapitops.plan.system.info.request.CacheAnalysisPageRequest;
|
||||
@ -15,16 +16,14 @@ import com.djrapitops.plan.system.info.request.CacheInspectPageRequest;
|
||||
import com.djrapitops.plan.system.info.request.CacheInspectPluginsTabRequest;
|
||||
import com.djrapitops.plan.system.info.request.CacheNetworkPageContentRequest;
|
||||
import com.djrapitops.plan.system.info.server.ServerInfo;
|
||||
import com.djrapitops.plan.utilities.Base64Util;
|
||||
import com.djrapitops.plan.utilities.MiscUtils;
|
||||
import com.djrapitops.plugin.api.TimeAmount;
|
||||
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.UUID;
|
||||
import java.util.*;
|
||||
|
||||
/**
|
||||
* Table that represents plan_transfer in SQL database.
|
||||
@ -33,54 +32,65 @@ import java.util.UUID;
|
||||
*/
|
||||
public class TransferTable extends Table {
|
||||
|
||||
private static final String columnSenderID = "sender_server_id";
|
||||
private static final String columnExpiry = "expiry_date";
|
||||
private static final String columnInfoType = "type";
|
||||
private static final String columnContent = "content_64";
|
||||
private static final String columnExtraVariables = "extra_variables";
|
||||
private final String insertStatementNoParts;
|
||||
|
||||
private final ServerTable serverTable;
|
||||
|
||||
private final String insertStatement;
|
||||
private final String insertStatementParts;
|
||||
private final String selectStatement;
|
||||
|
||||
public TransferTable(SQLDB db) {
|
||||
super("plan_transfer", db);
|
||||
|
||||
serverTable = db.getServerTable();
|
||||
insertStatement = "INSERT INTO " + tableName + " (" +
|
||||
columnSenderID + ", " +
|
||||
columnExpiry + ", " +
|
||||
columnInfoType + ", " +
|
||||
columnExtraVariables + ", " +
|
||||
columnContent +
|
||||
insertStatementParts = "REPLACE INTO " + tableName + " (" +
|
||||
Col.SENDER_ID + ", " +
|
||||
Col.EXPIRY + ", " +
|
||||
Col.INFO_TYPE + ", " +
|
||||
Col.EXTRA_VARIABLES + ", " +
|
||||
Col.CONTENT + ", " +
|
||||
Col.PART +
|
||||
") VALUES (" +
|
||||
serverTable.statementSelectServerID + ", " +
|
||||
"?, ?, ?, ?, ?)";
|
||||
insertStatementNoParts = "REPLACE INTO " + tableName + " (" +
|
||||
Col.SENDER_ID + ", " +
|
||||
Col.EXPIRY + ", " +
|
||||
Col.INFO_TYPE + ", " +
|
||||
Col.EXTRA_VARIABLES + ", " +
|
||||
Col.CONTENT +
|
||||
") VALUES (" +
|
||||
serverTable.statementSelectServerID + ", " +
|
||||
"?, ?, ?, ?)";
|
||||
|
||||
selectStatement = "SELECT * FROM " + tableName +
|
||||
" WHERE " + columnInfoType + "= ?" +
|
||||
" AND " + columnExpiry + "> ?" +
|
||||
" ORDER BY " + columnExpiry + " DESC";
|
||||
" WHERE " + Col.INFO_TYPE + "= ?" +
|
||||
" AND " + Col.EXPIRY + "> ?" +
|
||||
" ORDER BY " + Col.EXPIRY + " DESC, "
|
||||
+ Col.PART + " ASC";
|
||||
}
|
||||
|
||||
@Override
|
||||
public void createTable() throws DBInitException {
|
||||
createTable(TableSqlParser.createTable(tableName)
|
||||
.column(columnSenderID, Sql.INT).notNull()
|
||||
.column(columnExpiry, Sql.LONG).notNull().defaultValue("0")
|
||||
.column(columnInfoType, Sql.varchar(100)).notNull()
|
||||
.column(columnExtraVariables, Sql.varchar(255)).defaultValue("''")
|
||||
.column(columnContent, usingMySQL ? "MEDIUMTEXT" : Sql.varchar(1)) // SQLite does not enforce varchar limits.
|
||||
.foreignKey(columnSenderID, serverTable.toString(), serverTable.getColumnID())
|
||||
.column(Col.SENDER_ID, Sql.INT).notNull()
|
||||
.column(Col.EXPIRY, Sql.LONG).notNull().defaultValue("0")
|
||||
.column(Col.INFO_TYPE, Sql.varchar(100)).notNull()
|
||||
.column(Col.EXTRA_VARIABLES, Sql.varchar(255)).defaultValue("''")
|
||||
.column(Col.CONTENT, usingMySQL ? "MEDIUMTEXT" : Sql.varchar(1)) // SQLite does not enforce varchar limits.
|
||||
.column(Col.PART, Sql.LONG).notNull().defaultValue("0")
|
||||
.foreignKey(Col.SENDER_ID, serverTable.toString(), ServerTable.Col.SERVER_ID)
|
||||
.toString()
|
||||
);
|
||||
}
|
||||
|
||||
public void alterTableV14() {
|
||||
addColumns(Col.PART + " bigint NOT NULL DEFAULT 0");
|
||||
}
|
||||
|
||||
public void clean() throws SQLException {
|
||||
String sql = "DELETE FROM " + tableName +
|
||||
" WHERE " + columnExpiry + " < ?" +
|
||||
" AND " + columnInfoType + " != ?";
|
||||
" WHERE " + Col.EXPIRY + " < ?" +
|
||||
" AND " + Col.INFO_TYPE + " != ?";
|
||||
|
||||
execute(new ExecStatement(sql) {
|
||||
@Override
|
||||
@ -90,8 +100,8 @@ public class TransferTable extends Table {
|
||||
}
|
||||
});
|
||||
sql = "DELETE FROM " + tableName +
|
||||
" WHERE " + columnSenderID + " = " + serverTable.statementSelectServerID +
|
||||
" AND " + columnInfoType + " = ?";
|
||||
" WHERE " + Col.SENDER_ID + " = " + serverTable.statementSelectServerID +
|
||||
" AND " + Col.INFO_TYPE + " = ?";
|
||||
|
||||
execute(new ExecStatement(sql) {
|
||||
@Override
|
||||
@ -103,7 +113,7 @@ public class TransferTable extends Table {
|
||||
}
|
||||
|
||||
public void storePlayerHtml(UUID player, String encodedHtml) throws SQLException {
|
||||
execute(new ExecStatement(insertStatement) {
|
||||
execute(new ExecStatement(insertStatementNoParts) {
|
||||
@Override
|
||||
public void prepare(PreparedStatement statement) throws SQLException {
|
||||
statement.setString(1, ServerInfo.getServerUUID().toString());
|
||||
@ -116,20 +126,28 @@ public class TransferTable extends Table {
|
||||
}
|
||||
|
||||
public void storeServerHtml(UUID serverUUID, String encodedHtml) throws SQLException {
|
||||
execute(new ExecStatement(insertStatement) {
|
||||
@Override
|
||||
public void prepare(PreparedStatement statement) throws SQLException {
|
||||
statement.setString(1, ServerInfo.getServerUUID().toString());
|
||||
statement.setLong(2, MiscUtils.getTime() + TimeAmount.MINUTE.ms());
|
||||
statement.setString(3, CacheAnalysisPageRequest.class.getSimpleName().toLowerCase());
|
||||
statement.setString(4, serverUUID.toString());
|
||||
statement.setString(5, encodedHtml);
|
||||
}
|
||||
});
|
||||
List<String> split = Base64Util.split(encodedHtml, 500000L);
|
||||
|
||||
int i = 0;
|
||||
for (String part : split) {
|
||||
final int partNumber = i;
|
||||
execute(new ExecStatement(insertStatementParts) {
|
||||
@Override
|
||||
public void prepare(PreparedStatement statement) throws SQLException {
|
||||
statement.setString(1, ServerInfo.getServerUUID().toString());
|
||||
statement.setLong(2, MiscUtils.getTime() + TimeAmount.MINUTE.ms());
|
||||
statement.setString(3, CacheAnalysisPageRequest.class.getSimpleName().toLowerCase());
|
||||
statement.setString(4, serverUUID.toString());
|
||||
statement.setString(5, part);
|
||||
statement.setInt(6, partNumber);
|
||||
}
|
||||
});
|
||||
i++;
|
||||
}
|
||||
}
|
||||
|
||||
public void storeNetworkPageContent(UUID serverUUID, String encodedHtml) throws SQLException {
|
||||
execute(new ExecStatement(insertStatement) {
|
||||
execute(new ExecStatement(insertStatementNoParts) {
|
||||
@Override
|
||||
public void prepare(PreparedStatement statement) throws SQLException {
|
||||
statement.setString(1, ServerInfo.getServerUUID().toString());
|
||||
@ -153,18 +171,29 @@ public class TransferTable extends Table {
|
||||
public Map<UUID, String> processResults(ResultSet set) throws SQLException {
|
||||
Map<UUID, String> htmlPerUUID = new HashMap<>();
|
||||
while (set.next()) {
|
||||
String uuidString = set.getString(columnExtraVariables);
|
||||
String uuidString = set.getString(Col.EXTRA_VARIABLES.get());
|
||||
UUID uuid = UUID.fromString(uuidString);
|
||||
|
||||
if (!htmlPerUUID.containsKey(uuid)) {
|
||||
htmlPerUUID.put(uuid, set.getString(columnContent));
|
||||
}
|
||||
htmlPerUUID.put(uuid, htmlPerUUID.getOrDefault(uuid, "") + set.getString(Col.CONTENT.get()));
|
||||
}
|
||||
return htmlPerUUID;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
public void storePlayerPluginsTab(UUID player, String encodedHtml) throws SQLException {
|
||||
execute(new ExecStatement(insertStatementNoParts) {
|
||||
@Override
|
||||
public void prepare(PreparedStatement statement) throws SQLException {
|
||||
statement.setString(1, ServerInfo.getServerUUID().toString());
|
||||
statement.setLong(2, MiscUtils.getTime() + TimeAmount.MINUTE.ms());
|
||||
statement.setString(3, CacheInspectPluginsTabRequest.class.getSimpleName().toLowerCase());
|
||||
statement.setString(4, player.toString());
|
||||
statement.setString(5, encodedHtml);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
public Map<UUID, String> getPlayerHtml() throws SQLException {
|
||||
return getHtmlPerUUIDForCacheRequest(CacheInspectPageRequest.class);
|
||||
}
|
||||
@ -177,30 +206,17 @@ public class TransferTable extends Table {
|
||||
return getHtmlPerUUIDForCacheRequest(CacheAnalysisPageRequest.class);
|
||||
}
|
||||
|
||||
public void storePlayerPluginsTab(UUID player, String encodedHtml) throws SQLException {
|
||||
execute(new ExecStatement(insertStatement) {
|
||||
@Override
|
||||
public void prepare(PreparedStatement statement) throws SQLException {
|
||||
statement.setString(1, ServerInfo.getServerUUID().toString());
|
||||
statement.setLong(2, MiscUtils.getTime() + TimeAmount.MINUTE.ms());
|
||||
statement.setString(3, CacheInspectPluginsTabRequest.class.getSimpleName().toLowerCase());
|
||||
statement.setString(4, player.toString());
|
||||
statement.setString(5, encodedHtml);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
public Map<UUID, String> getPlayerPluginsTabs(UUID playerUUID) throws SQLException {
|
||||
String serverIDColumn = serverTable + "." + serverTable.getColumnID();
|
||||
String serverUUIDColumn = serverTable + "." + serverTable.getColumnUUID() + " as s_uuid";
|
||||
String sql = "SELECT " +
|
||||
columnContent + ", " +
|
||||
Col.CONTENT + ", " +
|
||||
serverUUIDColumn +
|
||||
" FROM " + tableName +
|
||||
" INNER JOIN " + serverTable + " on " + serverIDColumn + "=" + columnSenderID +
|
||||
" WHERE " + columnInfoType + "= ?" +
|
||||
" AND " + columnExpiry + "> ?" +
|
||||
" AND " + columnExtraVariables + "=?";
|
||||
" INNER JOIN " + serverTable + " on " + serverIDColumn + "=" + Col.SENDER_ID +
|
||||
" WHERE " + Col.INFO_TYPE + "= ?" +
|
||||
" AND " + Col.EXPIRY + "> ?" +
|
||||
" AND " + Col.EXTRA_VARIABLES + "=?";
|
||||
|
||||
return query(new QueryStatement<Map<UUID, String>>(sql, 250) {
|
||||
@Override
|
||||
@ -215,7 +231,7 @@ public class TransferTable extends Table {
|
||||
Map<UUID, String> htmlPerUUID = new HashMap<>();
|
||||
while (set.next()) {
|
||||
UUID serverUUID = UUID.fromString(set.getString("s_uuid"));
|
||||
String html64 = set.getString(columnContent);
|
||||
String html64 = set.getString(Col.CONTENT.get());
|
||||
|
||||
htmlPerUUID.put(serverUUID, html64);
|
||||
}
|
||||
@ -231,9 +247,9 @@ public class TransferTable extends Table {
|
||||
String sql = "SELECT " +
|
||||
serverUUIDColumn +
|
||||
" FROM " + tableName +
|
||||
" INNER JOIN " + serverTable + " on " + serverIDColumn + "=" + columnSenderID +
|
||||
" WHERE " + columnExtraVariables + "=?" +
|
||||
" ORDER BY " + columnExpiry + " LIMIT 1";
|
||||
" INNER JOIN " + serverTable + " on " + serverIDColumn + "=" + Col.SENDER_ID +
|
||||
" WHERE " + Col.EXTRA_VARIABLES + "=?" +
|
||||
" ORDER BY " + Col.EXPIRY + " LIMIT 1";
|
||||
|
||||
return query(new QueryStatement<Optional<UUID>>(sql, 1) {
|
||||
@Override
|
||||
@ -253,7 +269,7 @@ public class TransferTable extends Table {
|
||||
|
||||
@Deprecated
|
||||
public void storePlayerOnlineOnThisServer(UUID playerUUID) throws SQLException {
|
||||
execute(new ExecStatement(insertStatement) {
|
||||
execute(new ExecStatement(insertStatementNoParts) {
|
||||
@Override
|
||||
public void prepare(PreparedStatement statement) throws SQLException {
|
||||
statement.setString(1, ServerInfo.getServerUUID().toString());
|
||||
@ -266,7 +282,7 @@ public class TransferTable extends Table {
|
||||
}
|
||||
|
||||
public void storeConfigSettings(String encodedSettingString) throws SQLException {
|
||||
execute(new ExecStatement(insertStatement) {
|
||||
execute(new ExecStatement(insertStatementNoParts) {
|
||||
@Override
|
||||
public void prepare(PreparedStatement statement) throws SQLException {
|
||||
statement.setString(1, ServerInfo.getServerUUID().toString());
|
||||
@ -289,10 +305,34 @@ public class TransferTable extends Table {
|
||||
@Override
|
||||
public Optional<String> processResults(ResultSet set) throws SQLException {
|
||||
if (set.next()) {
|
||||
return Optional.ofNullable(set.getString(columnContent));
|
||||
return Optional.ofNullable(set.getString(Col.CONTENT.get()));
|
||||
}
|
||||
return Optional.empty();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
public enum Col implements Column {
|
||||
SENDER_ID("sender_server_id"),
|
||||
EXPIRY("expiry_date"),
|
||||
INFO_TYPE("type"),
|
||||
CONTENT("content_64"),
|
||||
EXTRA_VARIABLES("extra_variables"),
|
||||
PART("part");
|
||||
|
||||
private final String column;
|
||||
|
||||
Col(String column) {
|
||||
this.column = column;
|
||||
}
|
||||
|
||||
public String get() {
|
||||
return toString();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return column;
|
||||
}
|
||||
}
|
||||
}
|
@ -4,7 +4,9 @@
|
||||
*/
|
||||
package com.djrapitops.plan.utilities;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Base64;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Utility for performing Base64 operations.
|
||||
@ -29,4 +31,25 @@ public class Base64Util {
|
||||
return new String(decoded);
|
||||
}
|
||||
|
||||
public static List<String> split(String encoded, long partLength) {
|
||||
List<String> split = new ArrayList<>();
|
||||
StringBuilder builder = new StringBuilder();
|
||||
long length = 0;
|
||||
for (char c : encoded.toCharArray()) {
|
||||
builder.append(c);
|
||||
length++;
|
||||
if (length >= partLength) {
|
||||
split.add(builder.toString());
|
||||
builder = new StringBuilder();
|
||||
length = 0;
|
||||
}
|
||||
}
|
||||
|
||||
// Add the last part even if it isn't full length.
|
||||
if (length != 0) {
|
||||
split.add(builder.toString());
|
||||
}
|
||||
|
||||
return split;
|
||||
}
|
||||
}
|
@ -1,4 +1,4 @@
|
||||
name: Plan
|
||||
author: Rsl1122
|
||||
main: com.djrapitops.plan.PlanBungee
|
||||
version: 4.1.6-b2
|
||||
version: 4.1.6-b3
|
@ -1,7 +1,7 @@
|
||||
name: Plan
|
||||
author: Rsl1122
|
||||
main: com.djrapitops.plan.Plan
|
||||
version: 4.1.6-b2
|
||||
version: 4.1.6-b3
|
||||
softdepend:
|
||||
- EssentialsX
|
||||
- Towny
|
||||
|
@ -896,4 +896,27 @@ public class SQLiteTest {
|
||||
assertTrue(configSettings.isPresent());
|
||||
assertEquals(testString, Base64Util.decode(configSettings.get()));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testTransferSplitAndRearrange() throws SQLException {
|
||||
StringBuilder testData = new StringBuilder();
|
||||
for (int i = 0; i < 900000; i++) {
|
||||
testData.append("a");
|
||||
}
|
||||
|
||||
String testString = Base64Util.encode(testData.toString());
|
||||
|
||||
int length = testString.length();
|
||||
System.out.println("Test String Length: " + length);
|
||||
assertTrue(length > 500000);
|
||||
|
||||
System.out.println("Test Parts: " + (int) Math.ceil(length / 500000.0));
|
||||
|
||||
TransferTable transferTable = db.getTransferTable();
|
||||
transferTable.storeServerHtml(TestConstants.SERVER_UUID, testString);
|
||||
|
||||
String result = transferTable.getServerHtml().get(TestConstants.SERVER_UUID);
|
||||
assertNotNull(result);
|
||||
assertEquals(testString, result);
|
||||
}
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user