feat(backend): refactor pending versions to use object storage too

This commit is contained in:
MiniDigger | Martin 2022-12-29 23:04:23 +01:00
parent 4a934eadb9
commit e3520d5578
7 changed files with 95 additions and 123 deletions

View File

@ -13,6 +13,7 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.core.io.Resource;
import org.springframework.core.io.ResourceLoader;
import org.springframework.stereotype.Service;
import software.amazon.awssdk.services.s3.S3Client;
@Service
@ConditionalOnProperty(value = "hangar.storage.type", havingValue = "object")
@ -21,11 +22,13 @@ public class S3FileService implements FileService {
private final StorageConfig config;
private final ResourceLoader resourceLoader;
private final S3Template s3Template;
private final S3Client s3Client;
public S3FileService(final StorageConfig config, final ResourceLoader resourceLoader, final S3Template s3Template) {
public S3FileService(final StorageConfig config, final ResourceLoader resourceLoader, final S3Template s3Template, final S3Client s3Client) {
this.config = config;
this.resourceLoader = resourceLoader;
this.s3Template = s3Template;
this.s3Client = s3Client;
}
@Override
@ -39,8 +42,8 @@ public class S3FileService implements FileService {
}
@Override
public void deleteDirectory(final String dir) {
this.s3Template.deleteObject(this.config.bucket(), dir);
public void deleteDirectory(final String path) {
this.s3Template.deleteObject(path);
}
@Override
@ -64,7 +67,15 @@ public class S3FileService implements FileService {
@Override
public void move(final String oldPath, final String newPath) throws IOException {
if (!oldPath.startsWith(this.getRoot()) && newPath.startsWith(this.getRoot())) {
// upload from file to s3
this.write(Files.newInputStream(Path.of(oldPath)), newPath);
} else if (oldPath.startsWith(this.getRoot()) && newPath.startsWith(this.getRoot())) {
// "rename" in s3
this.s3Client.copyObject((builder -> builder
.sourceBucket(this.config.bucket()).sourceKey(oldPath.replace(this.getRoot(), ""))
.destinationBucket(this.config.bucket()).destinationKey(newPath.replace(this.getRoot(), ""))
));
this.s3Template.deleteObject(oldPath);
} else {
throw new UnsupportedOperationException("cant move " + oldPath + " to " + newPath);
}

View File

@ -1,36 +1,23 @@
package io.papermc.hangar.service.internal.uploads;
import io.papermc.hangar.config.hangar.StorageConfig;
import io.papermc.hangar.model.common.Platform;
import io.papermc.hangar.service.internal.file.FileService;
import io.papermc.hangar.util.FileUtils;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
@Component
public class ProjectFiles {
private static final Logger logger = LoggerFactory.getLogger(ProjectFiles.class);
private final String pluginsDir;
private final Path tmpDir;
private final String tmpDir;
private final FileService fileService;
@Autowired
public ProjectFiles(final StorageConfig storageConfig, final FileService fileService) {
public ProjectFiles(final FileService fileService) {
this.fileService = fileService;
final Path uploadsDir = Path.of(storageConfig.workDir());
this.pluginsDir = fileService.resolve(fileService.getRoot(), "plugins");
this.tmpDir = uploadsDir.resolve("tmp");
if (Files.exists(this.tmpDir)) {
FileUtils.deleteDirectory(this.tmpDir);
}
logger.info("Cleaned up tmp files and inited work dir {} ", uploadsDir);
this.tmpDir = fileService.resolve(fileService.getRoot(), "tmp");
}
public String getProjectDir(final String owner, final String slug) {
@ -91,8 +78,8 @@ public class ProjectFiles {
return this.fileService.resolve(this.getIconsDir(owner, slug), "icon.png");
}
public Path getTempDir(final String owner) {
return this.tmpDir.resolve(owner);
public String getTempDir(final String owner) {
return this.fileService.resolve(this.tmpDir, owner);
}
}

View File

@ -141,15 +141,13 @@ public class VersionFactory extends HangarComponent {
final Set<Platform> processedPlatforms = EnumSet.noneOf(Platform.class);
// Delete old temp data
final Path userTempDir = this.projectFiles.getTempDir(this.getHangarPrincipal().getName());
if (Files.exists(userTempDir)) {
FileUtils.deleteDirectory(userTempDir);
}
final String userTempDir = this.projectFiles.getTempDir(this.getHangarPrincipal().getName());
this.fileService.deleteDirectory(userTempDir);
for (final MultipartFileOrUrl fileOrUrl : data) {
// Make sure each platform only has one corresponding file/url
if (!processedPlatforms.addAll(fileOrUrl.platforms())) {
FileUtils.deleteDirectory(userTempDir);
this.fileService.deleteDirectory(userTempDir);
throw new IllegalArgumentException();
}
@ -161,8 +159,6 @@ public class VersionFactory extends HangarComponent {
}
}
this.tempDirCache.put(userTempDir, DUMMY);
for (final Platform platform : processedPlatforms) {
platformDependencies.putIfAbsent(platform, Collections.emptySortedSet());
pluginDependencies.putIfAbsent(platform, Collections.emptySet());
@ -170,41 +166,41 @@ public class VersionFactory extends HangarComponent {
return new PendingVersion(versionString, pluginDependencies, platformDependencies, pendingFiles, projectTable.isForumSync());
}
private String createPendingFile(final MultipartFile file, final String channel, final ProjectTable projectTable, final Map<Platform, Set<PluginDependency>> pluginDependencies, final Map<Platform, SortedSet<String>> platformDependencies, final List<PendingVersionFile> pendingFiles, String versionString, final Path userTempDir, final MultipartFileOrUrl fileOrUrl) {
private String createPendingFile(final MultipartFile file, final String channel, final ProjectTable projectTable, final Map<Platform, Set<PluginDependency>> pluginDependencies, final Map<Platform, SortedSet<String>> platformDependencies, final List<PendingVersionFile> pendingFiles, String versionString, final String userTempDir, final MultipartFileOrUrl fileOrUrl) {
// check extension
final String pluginFileName = file.getOriginalFilename();
if (pluginFileName == null || (!pluginFileName.endsWith(".zip") && !pluginFileName.endsWith(".jar"))) {
FileUtils.deleteDirectory(userTempDir);
this.fileService.deleteDirectory(userTempDir);
throw new HangarApiException(HttpStatus.BAD_REQUEST, "version.new.error.fileExtension");
}
// move file into temp
final PluginFileWithData pluginDataFile;
final Platform platformToResolve = fileOrUrl.platforms().get(0); // Just save it by the first platform
final Path tmpDir = userTempDir.resolve(platformToResolve.name());
try {
if (!Files.isDirectory(tmpDir)) {
Files.createDirectories(tmpDir);
}
final Platform platformToResolve = fileOrUrl.platforms().get(0); // Just save it by the first platform
// read file
final String tmpDir = this.fileService.resolve(userTempDir, platformToResolve.name());
final String tmpPluginFile = this.fileService.resolve(tmpDir, pluginFileName);
final byte[] bytes = file.getInputStream().readAllBytes();
// write
this.fileService.write(file.getInputStream(), tmpPluginFile);
// load meta
pluginDataFile = this.pluginDataService.loadMeta(pluginFileName, bytes, this.getHangarPrincipal().getUserId());
} catch (final ConfigurateException configurateException) {
this.logger.error("Error while reading file metadata while uploading {} for {}", pluginFileName, this.getHangarPrincipal().getName(), configurateException);
this.fileService.deleteDirectory(userTempDir);
throw new HangarApiException(HttpStatus.BAD_REQUEST, "version.new.error.metaNotFound");
} catch (final Exception e) {
this.logger.error("Error while uploading {} for {}", pluginFileName, this.getHangarPrincipal().getName(), e);
this.fileService.deleteDirectory(userTempDir);
throw new HangarApiException(HttpStatus.BAD_REQUEST, "version.new.error.unexpected");
}
final Path tmpPluginFile = tmpDir.resolve(pluginFileName);
file.transferTo(tmpPluginFile);
pluginDataFile = this.pluginDataService.loadMeta(tmpPluginFile, this.getHangarPrincipal().getUserId());
} catch (final ConfigurateException configurateException) {
this.logger.error("Error while reading file metadata while uploading {} for {}", pluginFileName, this.getHangarPrincipal().getName(), configurateException);
FileUtils.deleteDirectory(userTempDir);
throw new HangarApiException(HttpStatus.BAD_REQUEST, "version.new.error.metaNotFound");
} catch (final IOException e) {
this.logger.error("Error while uploading {} for {}", pluginFileName, this.getHangarPrincipal().getName(), e);
FileUtils.deleteDirectory(userTempDir);
throw new HangarApiException(HttpStatus.BAD_REQUEST, "version.new.error.unexpected");
if (versionString == null && pluginDataFile.data().getVersion() != null) {
versionString = StringUtils.slugify(pluginDataFile.data().getVersion());
}
if (versionString == null && pluginDataFile.getData().getVersion() != null) {
versionString = StringUtils.slugify(pluginDataFile.getData().getVersion());
}
final FileInfo fileInfo = new FileInfo(pluginDataFile.getPath().getFileName().toString(), pluginDataFile.getPath().toFile().length(), pluginDataFile.getMd5());
final FileInfo fileInfo = new FileInfo(pluginDataFile.fileName(), pluginDataFile.fileSize(), pluginDataFile.md5());
pendingFiles.add(new PendingVersionFile(fileOrUrl.platforms(), fileInfo, null));
// setup dependencies
@ -217,12 +213,12 @@ public class VersionFactory extends HangarComponent {
}
// If no previous version present, use uploaded version data
final Set<PluginDependency> loadedPluginDependencies = pluginDataFile.getData().getDependencies().get(platform);
final Set<PluginDependency> loadedPluginDependencies = pluginDataFile.data().getDependencies().get(platform);
if (loadedPluginDependencies != null) {
pluginDependencies.put(platform, loadedPluginDependencies);
}
final SortedSet<String> loadedPlatformDependencies = pluginDataFile.getData().getPlatformDependencies().get(platform);
final SortedSet<String> loadedPlatformDependencies = pluginDataFile.data().getPlatformDependencies().get(platform);
if (loadedPlatformDependencies != null) {
platformDependencies.put(platform, loadedPlatformDependencies);
}
@ -259,7 +255,7 @@ public class VersionFactory extends HangarComponent {
assert projectTable != null;
// verify that all platform files exist and that platform dependencies are setup correctly
final Path userTempDir = this.projectFiles.getTempDir(this.getHangarPrincipal().getName());
final String userTempDir = this.projectFiles.getTempDir(this.getHangarPrincipal().getName());
this.verifyPendingPlatforms(pendingVersion, userTempDir);
ProjectVersionTable projectVersionTable = null;
@ -368,7 +364,7 @@ public class VersionFactory extends HangarComponent {
this.projectVersionDependenciesDAO.insertAll(pluginDependencyTables);
}
private void processPendingVersionFile(final PendingVersion pendingVersion, final Path userTempDir, final ProjectVersionTable projectVersionTable, final String versionDir, final List<Pair<ProjectVersionDownloadTable, List<Platform>>> downloadsTables, final PendingVersionFile pendingVersionFile) throws IOException {
private void processPendingVersionFile(final PendingVersion pendingVersion, final String userTempDir, final ProjectVersionTable projectVersionTable, final String versionDir, final List<Pair<ProjectVersionDownloadTable, List<Platform>>> downloadsTables, final PendingVersionFile pendingVersionFile) throws IOException {
final FileInfo fileInfo = pendingVersionFile.fileInfo();
if (fileInfo == null) {
final ProjectVersionDownloadTable table = new ProjectVersionDownloadTable(projectVersionTable.getVersionId(), null, null, null, pendingVersionFile.externalUrl());
@ -378,10 +374,10 @@ public class VersionFactory extends HangarComponent {
// Move file for first platform
final Platform platformToResolve = pendingVersionFile.platforms().get(0);
final Path tmpVersionJar = userTempDir.resolve(platformToResolve.name()).resolve(fileInfo.getName());
final String tmpVersionJar = this.fileService.resolve(this.fileService.resolve(userTempDir, platformToResolve.name()), fileInfo.getName());
final String newVersionJarPath = this.fileService.resolve(this.fileService.resolve(versionDir, platformToResolve.name()), tmpVersionJar.getFileName().toString());
this.fileService.move(tmpVersionJar.toString(), newVersionJarPath);
final String newVersionJarPath = this.fileService.resolve(this.fileService.resolve(versionDir, platformToResolve.name()), fileInfo.getName());
this.fileService.move(tmpVersionJar, newVersionJarPath);
// Create links for the other platforms
for (int i = 1; i < pendingVersionFile.platforms().size(); i++) {
@ -391,7 +387,7 @@ public class VersionFactory extends HangarComponent {
}
final String platformPath = this.fileService.resolve(versionDir, platform.name());
final String platformJarPath = this.fileService.resolve(platformPath, tmpVersionJar.getFileName().toString());
final String platformJarPath = this.fileService.resolve(platformPath, fileInfo.getName());
if (this.fileService instanceof S3FileService) {
// this isn't nice, but we cant link, so what am I supposed to do?
// fileService.move(tmpVersionJar.toString(), platformJarPath);
@ -405,7 +401,7 @@ public class VersionFactory extends HangarComponent {
downloadsTables.add(ImmutablePair.of(table, pendingVersionFile.platforms()));
}
private void verifyPendingPlatforms(final PendingVersion pendingVersion, final Path userTempDir) {
private void verifyPendingPlatforms(final PendingVersion pendingVersion, final String userTempDir) {
final Set<Platform> processedPlatforms = EnumSet.noneOf(Platform.class);
for (final PendingVersionFile pendingVersionFile : pendingVersion.getFiles()) {
if (!processedPlatforms.addAll(pendingVersionFile.platforms())) {
@ -415,13 +411,15 @@ public class VersionFactory extends HangarComponent {
final FileInfo fileInfo = pendingVersionFile.fileInfo();
if (fileInfo != null) { // verify file
final Platform platform = pendingVersionFile.platforms().get(0); // Use the first platform to resolve
final Path tmpVersionJar = userTempDir.resolve(platform.name()).resolve(fileInfo.getName());
final String tmpVersionJar = this.fileService.resolve(this.fileService.resolve(userTempDir, platform.name()), fileInfo.getName());
try {
if (Files.notExists(tmpVersionJar)) {
if (!this.fileService.exists(tmpVersionJar)) {
throw new HangarApiException(HttpStatus.BAD_REQUEST, "version.new.error.noFile");
} else if (tmpVersionJar.toFile().length() != fileInfo.getSizeBytes()) {
}
final byte[] bytes = this.fileService.bytes(tmpVersionJar);
if (bytes.length != fileInfo.getSizeBytes()) {
throw new HangarApiException(HttpStatus.BAD_REQUEST, "version.new.error.mismatchedFileSize");
} else if (!Objects.equals(CryptoUtils.md5ToHex(Files.readAllBytes(tmpVersionJar)), fileInfo.getMd5Hash())) {
} else if (!Objects.equals(CryptoUtils.md5ToHex(bytes), fileInfo.getMd5Hash())) {
throw new HangarApiException(HttpStatus.BAD_REQUEST, "version.new.error.hashMismatch");
}
} catch (final IOException e) {

View File

@ -4,19 +4,19 @@ import io.papermc.hangar.exceptions.HangarApiException;
import io.papermc.hangar.model.common.Platform;
import io.papermc.hangar.service.internal.versions.plugindata.handler.FileTypeHandler;
import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.EnumMap;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.jar.JarEntry;
import java.util.jar.JarInputStream;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
import java.util.zip.ZipInputStream;
import io.papermc.hangar.util.CryptoUtils;
import org.jetbrains.annotations.NotNull;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
@ -33,15 +33,15 @@ public class PluginDataService {
}
}
public @NotNull PluginFileWithData loadMeta(final Path file, final long userId) throws IOException {
try (final JarInputStream jarInputStream = this.openJar(file)) {
public @NotNull PluginFileWithData loadMeta(final String fileName, final byte[] bytes, final long userId) throws IOException {
try (final Jar jar = this.openJar(fileName, new ByteArrayInputStream(bytes))) {
final Map<Platform, FileTypeHandler.FileData> fileDataMap = new EnumMap<>(Platform.class);
JarEntry jarEntry;
while ((jarEntry = jarInputStream.getNextJarEntry()) != null && fileDataMap.size() < Platform.getValues().length) {
while ((jarEntry = jar.stream().getNextJarEntry()) != null && fileDataMap.size() < Platform.getValues().length) {
final FileTypeHandler<?> fileTypeHandler = this.pluginFileTypeHandlers.get(jarEntry.getName());
if (fileTypeHandler != null) {
final BufferedReader reader = new BufferedReader(new InputStreamReader(jarInputStream));
final BufferedReader reader = new BufferedReader(new InputStreamReader(jar.stream()));
final FileTypeHandler.FileData fileData = fileTypeHandler.getData(reader);
fileDataMap.put(fileTypeHandler.getPlatform(), fileData);
}
@ -52,26 +52,33 @@ public class PluginDataService {
throw new HangarApiException("version.new.error.metaNotFound");
}
});
return new PluginFileWithData(file, new PluginFileData(fileDataMap), userId);
return new PluginFileWithData(jar.fileName(), bytes.length, CryptoUtils.md5ToHex(bytes), new PluginFileData(fileDataMap), userId);
}
}
public JarInputStream openJar(final Path file) throws IOException {
if (file.toString().endsWith(".jar")) {
return new JarInputStream(Files.newInputStream(file));
public Jar openJar(final String fileName, final InputStream file) throws IOException {
if (fileName.endsWith(".jar")) {
return new Jar(fileName, new JarInputStream(file));
} else {
final ZipFile zipFile = new ZipFile(file.toFile()); // gets closed by closing the input stream?
final Enumeration<? extends ZipEntry> entries = zipFile.entries();
final ZipInputStream stream = new ZipInputStream(file);
while (entries.hasMoreElements()) {
final ZipEntry zipEntry = entries.nextElement();
ZipEntry zipEntry;
while ((zipEntry = stream.getNextEntry()) != null) {
final String name = zipEntry.getName();
if (!zipEntry.isDirectory() && name.split("/").length == 1 && name.endsWith(".jar")) {
return new JarInputStream(zipFile.getInputStream(zipEntry));
// todo what about multiple jars in one zip?
return new Jar(zipEntry.getName(), new JarInputStream(stream));
}
}
throw new HangarApiException("version.new.error.jarNotFound");
}
}
record Jar(String fileName, JarInputStream stream) implements AutoCloseable {
@Override
public void close() throws IOException {
this.stream.close();
}
}
}

View File

@ -1,39 +1,3 @@
package io.papermc.hangar.service.internal.versions.plugindata;
import io.papermc.hangar.util.CryptoUtils;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
public class PluginFileWithData {
private final Path path;
private final PluginFileData data;
private final long userId;
public PluginFileWithData(final Path path, final PluginFileData data, final long userId) {
this.path = path;
this.data = data;
this.userId = userId;
}
public Path getPath() {
return this.path;
}
public PluginFileData getData() {
return this.data;
}
public long getUserId() {
return this.userId;
}
public String getMd5() {
try {
return CryptoUtils.md5ToHex(Files.readAllBytes(this.path));
} catch (final IOException e) {
e.printStackTrace();
return null;
}
}
}
public record PluginFileWithData(String fileName, int fileSize, String md5, PluginFileData data, long userId) {}

View File

@ -6,6 +6,7 @@ import io.papermc.hangar.model.common.Platform;
import io.papermc.hangar.service.internal.versions.plugindata.handler.PaperFileTypeHandler;
import io.papermc.hangar.service.internal.versions.plugindata.handler.VelocityFileTypeHandler;
import io.papermc.hangar.service.internal.versions.plugindata.handler.WaterfallFileTypeHandler;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import java.util.Set;
@ -33,7 +34,7 @@ class PluginDataServiceTest {
@Test
void testLoadPaperPluginMetadata() throws Exception {
final PluginFileData data = this.classUnderTest.loadMeta(path.resolve("Paper.jar"), -1).getData();
final PluginFileData data = this.classUnderTest.loadMeta("Paper.jar", Files.newInputStream(path.resolve("Paper.jar")).readAllBytes(), -1).data();
data.validate();
assertEquals("Maintenance", data.getName());
assertEquals("Enable maintenance mode with a custom maintenance motd and icon.", data.getDescription());
@ -50,7 +51,7 @@ class PluginDataServiceTest {
@Test
void testLoadWaterfallPluginMetadata() throws Exception {
final PluginFileData data = this.classUnderTest.loadMeta(path.resolve("Waterfall.jar"), -1).getData();
final PluginFileData data = this.classUnderTest.loadMeta("Waterfall.jar", Files.newInputStream(path.resolve("Waterfall.jar")).readAllBytes(), -1).data();
data.validate();
assertEquals("Maintenance", data.getName());
@ -67,7 +68,7 @@ class PluginDataServiceTest {
@Test
void testLoadVelocityPluginMetadata() throws Exception {
final PluginFileData data = this.classUnderTest.loadMeta(path.resolve("Velocity.jar"), -1).getData();
final PluginFileData data = this.classUnderTest.loadMeta("Velocity.jar", Files.newInputStream(path.resolve("Velocity.jar")).readAllBytes(), -1).data();
data.validate();
assertEquals("Maintenance", data.getName());
@ -83,7 +84,7 @@ class PluginDataServiceTest {
@Test
void testLoadPaperPluginZipMetadata() throws Exception {
final PluginFileData data = this.classUnderTest.loadMeta(path.resolve("TestZip.zip"), -1).getData();
final PluginFileData data = this.classUnderTest.loadMeta("TestZip.zip", Files.newInputStream(path.resolve("TestZip.zip")).readAllBytes(), -1).data();
data.validate();
assertEquals("Maintenance", data.getName());
@ -107,7 +108,7 @@ class PluginDataServiceTest {
void testLoadMetaShouldFail(final String jarName, final String expectedMsg) {
final Path jarPath = path.resolve(jarName);
final HangarApiException exception = assertThrows(HangarApiException.class, () -> {
this.classUnderTest.loadMeta(jarPath, -1);
this.classUnderTest.loadMeta(jarName, Files.newInputStream(jarPath).readAllBytes(), -1);
});
assertEquals("400 BAD_REQUEST \"" + expectedMsg + "\"", exception.getMessage());
}

View File

@ -17,3 +17,7 @@ fake-user:
enabled: true
username: test
email: test@papermc.io
hangar:
storage:
type: "local"