IP storage updated for GDPR compliance

Plan/src/main/java/com/djrapitops/plan/data/PlayerProfile.java

@@ -241,7 +241,7 @@ public class PlayerProfile {
 
     public GeoInfo getMostRecentGeoInfo() {
         if (geoInformation.isEmpty()) {
-            return new GeoInfo("-", "Not Known", MiscUtils.getTime());
+            return new GeoInfo("-", "Not Known", MiscUtils.getTime(), "");
         }
         geoInformation.sort(new GeoInfoComparator());
         return geoInformation.get(0);

Plan/src/main/java/com/djrapitops/plan/data/container/GeoInfo.java

@@ -4,8 +4,13 @@
  */
 package com.djrapitops.plan.data.container;
 
+import com.djrapitops.plan.utilities.FormatUtils;
+import com.djrapitops.plan.utilities.SHA256Hash;
 import com.google.common.base.Objects;
 
+import java.io.UnsupportedEncodingException;
+import java.security.NoSuchAlgorithmException;
+
 /**
  * Data class that contains information about IP and Geolocation.
  *
@@ -15,12 +20,19 @@ public class GeoInfo {
 
     private final String ip;
     private final String geolocation;
+    private final String ipHash;
     private final long lastUsed;
 
-    public GeoInfo(String ip, String geolocation, long lastUsed) {
+    public GeoInfo(String ip, String geolocation, long lastUsed)
+            throws UnsupportedEncodingException, NoSuchAlgorithmException {
+        this(FormatUtils.formatIP(ip), geolocation, lastUsed, new SHA256Hash(ip).create());
+    }
+
+    public GeoInfo(String ip, String geolocation, long lastUsed, String ipHash) {
         this.ip = ip;
         this.geolocation = geolocation;
         this.lastUsed = lastUsed;
+        this.ipHash = ipHash;
     }
 
     public String getIp() {
@@ -35,6 +47,10 @@ public class GeoInfo {
         return lastUsed;
     }
 
+    public String getIpHash() {
+        return ipHash;
+    }
+
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;

Plan/src/main/java/com/djrapitops/plan/system/database/databases/sql/SQLDB.java

@@ -133,7 +133,7 @@ public abstract class SQLDB extends Database {
 
             if (newDatabase) {
                 Log.info("New Database created.");
-                versionTable.setVersion(16);
+                versionTable.setVersion(17);
             }
 
             int version = versionTable.getVersion();
@@ -177,6 +177,10 @@ public abstract class SQLDB extends Database {
                 worldTimesTable.alterTableV16();
                 versionTable.setVersion(16);
             }
+            if (version < 17) {
+                geoInfoTable.alterTableV17();
+                versionTable.setVersion(17);
+            }
         } catch (SQLException e) {
             throw new DBInitException("Failed to set-up Database", e);
         }

Plan/src/main/java/com/djrapitops/plan/system/database/databases/sql/tables/GeoInfoTable.java

@@ -10,8 +10,14 @@ import com.djrapitops.plan.system.database.databases.sql.statements.Column;
 import com.djrapitops.plan.system.database.databases.sql.statements.Select;
 import com.djrapitops.plan.system.database.databases.sql.statements.Sql;
 import com.djrapitops.plan.system.database.databases.sql.statements.TableSqlParser;
+import com.djrapitops.plan.system.settings.Settings;
+import com.djrapitops.plugin.api.utility.log.Log;
+import com.djrapitops.plugin.task.AbsRunnable;
+import com.djrapitops.plugin.task.RunnableFactory;
 import com.djrapitops.plugin.utilities.Verify;
 
+import java.io.UnsupportedEncodingException;
+import java.security.NoSuchAlgorithmException;
 import java.sql.PreparedStatement;
 import java.sql.ResultSet;
 import java.sql.SQLException;
@@ -33,11 +39,12 @@ public class GeoInfoTable extends UserIDTable {
         insertStatement = "INSERT INTO " + tableName + " ("
                 + Col.USER_ID + ", "
                 + Col.IP + ", "
+                + Col.IP_HASH + ", "
                 + Col.GEOLOCATION + ", "
                 + Col.LAST_USED
                 + ") VALUES ("
                 + usersTable.statementSelectID + ", "
-                + "?, ?, ?)";
+                + "?, ?, ?, ?)";
     }
 
     private String insertStatement;
@@ -48,6 +55,7 @@ public class GeoInfoTable extends UserIDTable {
                 .column(Col.USER_ID, Sql.INT).notNull()
                 .column(Col.IP, Sql.varchar(39)).notNull()
                 .column(Col.GEOLOCATION, Sql.varchar(50)).notNull()
+                .column(Col.IP_HASH, Sql.varchar(200))
                 .column(Col.LAST_USED, Sql.LONG).notNull().defaultValue("0")
                 .foreignKey(Col.USER_ID, usersTable.getTableName(), UsersTable.Col.ID)
                 .toString()
@@ -64,6 +72,49 @@ public class GeoInfoTable extends UserIDTable {
         addColumns(Col.LAST_USED + " bigint NOT NULL DEFAULT 0");
     }
 
+    public void alterTableV17() {
+        addColumns(Col.IP_HASH.get() + " varchar(200) DEFAULT ''");
+
+        RunnableFactory.createNew("DB Version 16->17", new AbsRunnable() {
+            @Override
+            public void run() {
+                try {
+                    Map<UUID, List<GeoInfo>> allGeoInfo = getAllGeoInfo();
+
+                    String sql = "UPDATE " + tableName + " SET " +
+                            Col.IP + "=?, " +
+                            Col.IP_HASH + "=? " +
+                            "WHERE " + Col.IP + "=?";
+                    executeBatch(new ExecStatement(sql) {
+                        @Override
+                        public void prepare(PreparedStatement statement) throws SQLException {
+                            for (List<GeoInfo> geoInfos : allGeoInfo.values()) {
+                                for (GeoInfo geoInfo : geoInfos) {
+                                    try {
+                                        GeoInfo updatedInfo = new GeoInfo(
+                                                geoInfo.getIp(),
+                                                geoInfo.getGeolocation(),
+                                                geoInfo.getLastUsed()
+                                        );
+                                        statement.setString(1, updatedInfo.getIp());
+                                        statement.setString(2, updatedInfo.getIpHash());
+                                        statement.setString(3, geoInfo.getIp());
+                                    } catch (UnsupportedEncodingException | NoSuchAlgorithmException e) {
+                                        if (Settings.DEV_MODE.isTrue()) {
+                                            Log.toLog(this.getClass(), e);
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                    });
+                } catch (SQLException e) {
+                    e.printStackTrace();
+                }
+            }
+        });
+    }
+
     public List<GeoInfo> getGeoInfo(UUID uuid) throws SQLException {
         String sql = "SELECT DISTINCT * FROM " + tableName +
                 " WHERE " + Col.USER_ID + "=" + usersTable.statementSelectID;
@@ -80,8 +131,9 @@ public class GeoInfoTable extends UserIDTable {
                 while (set.next()) {
                     String ip = set.getString(Col.IP.get());
                     String geolocation = set.getString(Col.GEOLOCATION.get());
+                    String ipHash = set.getString(Col.IP_HASH.get());
                     long lastUsed = set.getLong(Col.LAST_USED.get());
-                    geoInfo.add(new GeoInfo(ip, geolocation, lastUsed));
+                    geoInfo.add(new GeoInfo(ip, geolocation, lastUsed, ipHash));
                 }
                 return geoInfo;
             }
@@ -92,7 +144,7 @@ public class GeoInfoTable extends UserIDTable {
         String sql = "UPDATE " + tableName + " SET "
                 + Col.LAST_USED + "=?" +
                 " WHERE " + Col.USER_ID + "=" + usersTable.statementSelectID +
-                " AND " + Col.IP + "=?" +
+                " AND " + Col.IP_HASH + "=?" +
                 " AND " + Col.GEOLOCATION + "=?";
 
         execute(new ExecStatement(sql) {
@@ -100,7 +152,7 @@ public class GeoInfoTable extends UserIDTable {
             public void prepare(PreparedStatement statement) throws SQLException {
                 statement.setLong(1, info.getLastUsed());
                 statement.setString(2, uuid.toString());
-                statement.setString(3, info.getIp());
+                statement.setString(3, info.getIpHash());
                 statement.setString(4, info.getGeolocation());
             }
         });
@@ -120,8 +172,9 @@ public class GeoInfoTable extends UserIDTable {
             public void prepare(PreparedStatement statement) throws SQLException {
                 statement.setString(1, uuid.toString());
                 statement.setString(2, info.getIp());
-                statement.setString(3, info.getGeolocation());
+                statement.setString(3, info.getIpHash());
-                statement.setLong(4, info.getLastUsed());
+                statement.setString(4, info.getGeolocation());
+                statement.setLong(5, info.getLastUsed());
             }
         });
     }
@@ -154,6 +207,7 @@ public class GeoInfoTable extends UserIDTable {
                 Col.IP + ", " +
                 Col.GEOLOCATION + ", " +
                 Col.LAST_USED + ", " +
+                Col.IP_HASH + ", " +
                 usersUUIDColumn +
                 " FROM " + tableName +
                 " INNER JOIN " + usersTable + " on " + usersIDColumn + "=" + Col.USER_ID;
@@ -169,8 +223,9 @@ public class GeoInfoTable extends UserIDTable {
 
                     String ip = set.getString(Col.IP.get());
                     String geolocation = set.getString(Col.GEOLOCATION.get());
+                    String ipHash = set.getString(Col.IP_HASH.get());
                     long lastUsed = set.getLong(Col.LAST_USED.get());
-                    userGeoInfo.add(new GeoInfo(ip, geolocation, lastUsed));
+                    userGeoInfo.add(new GeoInfo(ip, geolocation, lastUsed, ipHash));
 
                     geoLocations.put(uuid, userGeoInfo);
                 }
@@ -203,9 +258,40 @@ public class GeoInfoTable extends UserIDTable {
         });
     }
 
+    public void insertAllGeoInfo(Map<UUID, List<GeoInfo>> allIPsAndGeolocations) throws SQLException {
+        if (Verify.isEmpty(allIPsAndGeolocations)) {
+            return;
+        }
+
+        executeBatch(new ExecStatement(insertStatement) {
+            @Override
+            public void prepare(PreparedStatement statement) throws SQLException {
+                // Every User
+                for (UUID uuid : allIPsAndGeolocations.keySet()) {
+                    // Every GeoInfo
+                    for (GeoInfo info : allIPsAndGeolocations.get(uuid)) {
+                        String ip = info.getIp();
+                        String ipHash = info.getIpHash();
+                        String geoLocation = info.getGeolocation();
+                        long lastUsed = info.getLastUsed();
+
+                        statement.setString(1, uuid.toString());
+                        statement.setString(2, ip);
+                        statement.setString(3, ipHash);
+                        statement.setString(4, geoLocation);
+                        statement.setLong(5, lastUsed);
+
+                        statement.addBatch();
+                    }
+                }
+            }
+        });
+    }
+
     public enum Col implements Column {
         USER_ID(UserIDTable.Col.USER_ID.get()),
         IP("ip"),
+        IP_HASH("ip_hash"),
         GEOLOCATION("geolocation"),
         LAST_USED("last_used");
 
@@ -225,32 +311,4 @@ public class GeoInfoTable extends UserIDTable {
             return column;
         }
     }
-
-    public void insertAllGeoInfo(Map<UUID, List<GeoInfo>> allIPsAndGeolocations) throws SQLException {
-        if (Verify.isEmpty(allIPsAndGeolocations)) {
-            return;
-        }
-
-        executeBatch(new ExecStatement(insertStatement) {
-            @Override
-            public void prepare(PreparedStatement statement) throws SQLException {
-                // Every User
-                for (UUID uuid : allIPsAndGeolocations.keySet()) {
-                    // Every GeoInfo
-                    for (GeoInfo info : allIPsAndGeolocations.get(uuid)) {
-                        String ip = info.getIp();
-                        String geoLocation = info.getGeolocation();
-                        long lastUsed = info.getLastUsed();
-
-                        statement.setString(1, uuid.toString());
-                        statement.setString(2, ip);
-                        statement.setString(3, geoLocation);
-                        statement.setLong(4, lastUsed);
-
-                        statement.addBatch();
-                    }
-                }
-            }
-        });
-    }
 }

Plan/src/main/java/com/djrapitops/plan/system/processing/importing/importers/Importer.java

@@ -23,6 +23,8 @@ import com.djrapitops.plugin.api.utility.log.Log;
 import com.djrapitops.plugin.utilities.Verify;
 import com.google.common.collect.ImmutableMap;
 
+import java.io.UnsupportedEncodingException;
+import java.security.NoSuchAlgorithmException;
 import java.util.*;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
@@ -228,7 +230,11 @@ public abstract class Importer {
         return userImportData.getIps().parallelStream()
                 .map(ip -> {
                     String geoLoc = GeolocationCache.getCountry(ip);
-                    return new GeoInfo(ip, geoLoc, date);
+                    try {
+                        return new GeoInfo(ip, geoLoc, date);
+                    } catch (UnsupportedEncodingException | NoSuchAlgorithmException e) {
+                        throw new IllegalArgumentException(e);
+                    }
                 }).collect(Collectors.toList());
     }
 

Plan/src/main/java/com/djrapitops/plan/system/processing/processors/player/IPUpdateProcessor.java

@@ -13,6 +13,8 @@ import com.djrapitops.plan.system.settings.Settings;
 import com.djrapitops.plugin.api.Check;
 import com.djrapitops.plugin.api.utility.log.Log;
 
+import java.io.UnsupportedEncodingException;
+import java.security.NoSuchAlgorithmException;
 import java.util.UUID;
 
 /**
@@ -38,7 +40,7 @@ public class IPUpdateProcessor implements CriticalRunnable {
             String country = GeolocationCache.getCountry(ip);
             try {
                 Database.getActive().save().geoInfo(uuid, new GeoInfo(ip, country, time));
-            } catch (DBException e) {
+            } catch (DBException | UnsupportedEncodingException | NoSuchAlgorithmException e) {
                 Log.toLog(this.getClass(), e);
             }
         }

Plan/src/main/java/com/djrapitops/plan/utilities/FormatUtils.java

@@ -276,10 +276,13 @@ public class FormatUtils {
     }
 
     public static String formatIP(String ip) {
+        if ("localhost".equals(ip)) {
+            return ip;
+        }
         StringBuilder b = new StringBuilder();
         int i = 0;
         for (String part : ip.split("\\.")) {
-            if (i >= 3) {
+            if (i >= 2) {
                 break;
             }
 
@@ -288,7 +291,7 @@ public class FormatUtils {
             i++;
         }
 
-        return b.append("xx").toString();
+        return b.append("xx.xx").toString();
     }
 
     /**

Plan/src/main/java/com/djrapitops/plan/utilities/SHA256Hash.java

@@ -0,0 +1,21 @@
+package com.djrapitops.plan.utilities;
+
+import java.io.UnsupportedEncodingException;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.Base64;
+
+public class SHA256Hash {
+
+    private final String original;
+
+    public SHA256Hash(String original) {
+        this.original = original;
+    }
+
+    public String create() throws NoSuchAlgorithmException, UnsupportedEncodingException {
+        MessageDigest digest = MessageDigest.getInstance("SHA-256");
+        digest.update(original.getBytes("UTF-8"));
+        return Base64.getEncoder().encodeToString(digest.digest());
+    }
+}

Plan/src/test/java/com/djrapitops/plan/system/database/databases/SQLiteTest.java

@@ -22,6 +22,7 @@ import com.djrapitops.plan.system.processing.processors.player.RegisterProcessor
 import com.djrapitops.plan.utilities.Base64Util;
 import com.djrapitops.plan.utilities.ManageUtils;
 import com.djrapitops.plan.utilities.MiscUtils;
+import com.djrapitops.plan.utilities.SHA256Hash;
 import com.djrapitops.plan.utilities.analysis.MathUtils;
 import com.djrapitops.plugin.StaticHolder;
 import com.djrapitops.plugin.api.utility.log.Log;
@@ -36,8 +37,10 @@ import utilities.TestConstants;
 import utilities.TestErrorManager;
 import utilities.mocks.SystemMockUtil;
 
+import java.io.UnsupportedEncodingException;
 import java.lang.management.ManagementFactory;
 import java.lang.management.OperatingSystemMXBean;
+import java.security.NoSuchAlgorithmException;
 import java.sql.SQLException;
 import java.util.*;
 
@@ -258,7 +261,7 @@ public class SQLiteTest {
         String expectedGeoLoc = "TestLocation";
         long time = MiscUtils.getTime();
 
-        GeoInfo expected = new GeoInfo(expectedIP, expectedGeoLoc, time);
+        GeoInfo expected = new GeoInfo(expectedIP, expectedGeoLoc, time, "3");
         geoInfoTable.saveGeoInfo(playerUUID, expected);
         geoInfoTable.saveGeoInfo(playerUUID, expected);
         commitTest();
@@ -563,7 +566,7 @@ public class SQLiteTest {
 
         sessionsTable.saveSession(playerUUID, session);
         nicknamesTable.saveUserName(playerUUID, "TestNick");
-        geoInfoTable.saveGeoInfo(playerUUID, new GeoInfo("1.2.3.4", "TestLoc", 223456789L));
+        geoInfoTable.saveGeoInfo(playerUUID, new GeoInfo("1.2.3.4", "TestLoc", 223456789L, "3"));
         actionsTable.insertAction(playerUUID, new Action(1324L, Actions.FIRST_SESSION, "Add"));
 
         assertTrue(usersTable.isRegistered(playerUUID));
@@ -579,7 +582,7 @@ public class SQLiteTest {
     }
 
     @Test
-    public void testRemovalEverything() throws SQLException, DBException {
+    public void testRemovalEverything() throws SQLException, DBException, UnsupportedEncodingException, NoSuchAlgorithmException {
         UserInfoTable userInfoTable = db.getUserInfoTable();
         UsersTable usersTable = db.getUsersTable();
         SessionsTable sessionsTable = db.getSessionsTable();
@@ -608,7 +611,7 @@ public class SQLiteTest {
         assertTrue(securityTable.getUsers().isEmpty());
     }
 
-    private void saveAllData(SQLDB database) throws SQLException {
+    private void saveAllData(SQLDB database) throws SQLException, UnsupportedEncodingException, NoSuchAlgorithmException {
         System.out.println("Saving all possible data to the Database..");
         UserInfoTable userInfoTable = database.getUserInfoTable();
         UsersTable usersTable = database.getUsersTable();
@@ -632,7 +635,8 @@ public class SQLiteTest {
 
         sessionsTable.saveSession(playerUUID, session);
         nicknamesTable.saveUserName(playerUUID, "TestNick");
-        geoInfoTable.saveGeoInfo(playerUUID, new GeoInfo("1.2.3.4", "TestLoc", 223456789L));
+        geoInfoTable.saveGeoInfo(playerUUID, new GeoInfo("1.2.3.4", "TestLoc", 223456789L,
+                new SHA256Hash("1.2.3.4").create()));
         actionsTable.insertAction(playerUUID, new Action(1324L, Actions.FIRST_SESSION, "Add"));
 
         assertTrue(usersTable.isRegistered(playerUUID));
@@ -756,7 +760,7 @@ public class SQLiteTest {
     }
 
     @Test
-    public void testBackupAndRestore() throws SQLException, DBInitException {
+    public void testBackupAndRestore() throws SQLException, DBInitException, UnsupportedEncodingException, NoSuchAlgorithmException {
         System.out.println("- Creating Backup Database -");
         SQLiteDB backup = new SQLiteDB("debug-backup" + MiscUtils.getTime());
         backup.init();
@@ -894,7 +898,7 @@ public class SQLiteTest {
     }
 
     @Test
-    public void testWorldTableGetWorldNamesNoException() throws SQLException {
+    public void testWorldTableGetWorldNamesNoException() throws SQLException, UnsupportedEncodingException, NoSuchAlgorithmException {
         saveAllData(db);
         Set<String> worldNames = db.getWorldTable().getWorldNames(TestConstants.SERVER_UUID);
         assertEquals(new HashSet<>(worlds), worldNames);
@@ -924,11 +928,11 @@ public class SQLiteTest {
         usersTable.registerUser(secondUuid, 0, "");
         usersTable.registerUser(thirdUuid, 0, "");
 
-        geoInfoTable.saveGeoInfo(firstUuid, new GeoInfo("-", "Test1", 0));
+        geoInfoTable.saveGeoInfo(firstUuid, new GeoInfo("-", "Test1", 0, "3"));
-        GeoInfo secondInfo = new GeoInfo("-", "Test2", 5);
+        GeoInfo secondInfo = new GeoInfo("-", "Test2", 5, "3");
         geoInfoTable.saveGeoInfo(firstUuid, secondInfo);
-        geoInfoTable.saveGeoInfo(secondUuid, new GeoInfo("-", "Test3", 0));
+        geoInfoTable.saveGeoInfo(secondUuid, new GeoInfo("-", "Test3", 0, "3"));
-        geoInfoTable.saveGeoInfo(thirdUuid, new GeoInfo("-", "Test4", 0));
+        geoInfoTable.saveGeoInfo(thirdUuid, new GeoInfo("-", "Test4", 0, "3"));
 
         List<String> geolocations = geoInfoTable.getNetworkGeolocations();
         System.out.println(geolocations);

Plan/src/test/java/com/djrapitops/plan/utilities/FormatUtilsTest.java

@@ -161,7 +161,7 @@ public class FormatUtilsTest {
         String ip = "1.2.3.4";
         String ip2 = "1.2.3.26";
         String ip3 = "1.2.3.235";
-        String expected = "1.2.3.xx";
+        String expected = "1.2.xx.xx";
 
         assertEquals(expected, FormatUtils.formatIP(ip));
         assertEquals(expected, FormatUtils.formatIP(ip2));

Plan/src/test/java/com/djrapitops/plan/utilities/SHA256HashTest.java

@@ -0,0 +1,19 @@
+package com.djrapitops.plan.utilities;
+
+import org.junit.Test;
+
+import java.io.UnsupportedEncodingException;
+import java.security.NoSuchAlgorithmException;
+
+import static org.junit.Assert.assertEquals;
+
+public class SHA256HashTest {
+
+    @Test
+    public void sameStringReturnsSameHash() throws UnsupportedEncodingException, NoSuchAlgorithmException {
+        String expected = new SHA256Hash("1.3.4.5").create();
+        String result = new SHA256Hash("1.3.4.5").create();
+        assertEquals(expected, result);
+    }
+
+}