perf: move version downloads to version queries

This commit is contained in:
MiniDigger | Martin 2025-02-02 16:41:18 +01:00
parent 248081c790
commit d80510b9b1
10 changed files with 120 additions and 70 deletions

View File

@ -43,13 +43,6 @@ public interface ProjectVersionsDAO {
@SqlQuery("SELECT * FROM project_versions WHERE project_id = :projectId AND version_string = :versionString")
ProjectVersionTable getProjectVersion(long projectId, String versionString);
@SqlQuery("SELECT pv.*" +
" FROM project_versions pv" +
" JOIN project_channels pc ON pv.channel_id = pc.id" +
" WHERE pc.id = :channelId" +
" ORDER BY pv.created_at DESC")
ProjectVersionTable getLastVersionOnChannel(long channelId);
@SqlQuery("SELECT pv.* FROM project_versions pv" +
" JOIN project_version_platform_dependencies pvpd ON pv.id = pvpd.version_id" +
" JOIN platform_versions v ON pvpd.platform_version_id = v.id" +
@ -68,13 +61,4 @@ public interface ProjectVersionsDAO {
" pv.version_string = :versionString" +
" LIMIT 1")
ProjectVersionTable getProjectVersionTableWithProjectSlug(String slug, String versionString);
@SingleValue
@UseEnumStrategy(EnumStrategy.BY_ORDINAL)
@SqlQuery("SELECT array_agg(DISTINCT plv.platform)" +
" FROM project_versions pv" +
" JOIN project_version_platform_dependencies pvpd ON pv.id = pvpd.version_id" +
" JOIN platform_versions plv ON pvpd.platform_version_id = plv.id" +
" WHERE pv.id = :versionId GROUP BY pv.id")
List<Platform> getVersionPlatforms(long versionId);
}

View File

@ -40,7 +40,8 @@ public interface VersionsApiDAO {
pv.description,
coalesce((SELECT sum(pvd.downloads) FROM project_versions_downloads pvd WHERE pv.id = pvd.version_id), 0) vs_totalDownloads,
(select array_agg(d) from (SELECT pvd.platform, sum(pvd.downloads) FROM project_versions_downloads pvd WHERE pv.id = pvd.version_id GROUP BY pvd.platform) d) vs_platformDownloads,
u.name author,
(SELECT ARRAY[p.owner_name, p.slug] FROM projects p WHERE p.id = pv.project_id limit 1) AS project_namespace,
(select u.name from users u where u.id = pv.author_id) as author,
pv.review_state,
pc.created_at pc_created_at,
pc.name pc_name,
@ -51,10 +52,18 @@ public interface VersionsApiDAO {
WHEN exists(SELECT * FROM pinned_versions piv WHERE piv.version_id = pv.id AND lower(type) = 'channel') THEN 'CHANNEL'
WHEN exists(SELECT * FROM pinned_versions piv WHERE piv.version_id = pv.id AND lower(type) = 'version') THEN 'VERSION'
ELSE 'NONE'
END AS pinnedStatus
END AS pinnedStatus,
(SELECT json_agg(json_build_object('file_size', file_size,
'hash', hash,
'file_name', file_name,
'external_url', external_url,
'platforms', platforms,
'download_platform', download_platform)) AS value
FROM project_version_downloads
WHERE version_id = pv.id
GROUP BY version_id) AS downloads
FROM project_versions pv
JOIN project_channels pc ON pv.channel_id = pc.id
LEFT JOIN users u ON pv.author_id = u.id
WHERE
<if(!canSeeHidden)>
(pv.visibility = 0
@ -84,6 +93,7 @@ public interface VersionsApiDAO {
pv.description,
coalesce((SELECT sum(pvd.downloads) FROM project_versions_downloads pvd WHERE pv.id = pvd.version_id), 0) vs_totalDownloads,
(select array_agg(d) from (SELECT pvd.platform, sum(pvd.downloads) FROM project_versions_downloads pvd WHERE pv.id = pvd.version_id GROUP BY pvd.platform) d) vs_platformDownloads,
(SELECT ARRAY[p.owner_name, p.slug] FROM projects p WHERE p.id = pv.project_id limit 1) AS project_namespace,
(select u.name from users u where u.id = pv.author_id) as author,
pv.review_state,
pc.created_at pc_created_at,
@ -95,7 +105,16 @@ public interface VersionsApiDAO {
WHEN exists(SELECT * FROM pinned_versions piv WHERE piv.version_id = pv.id AND lower(type) = 'channel') THEN 'CHANNEL'
WHEN exists(SELECT * FROM pinned_versions piv WHERE piv.version_id = pv.id AND lower(type) = 'version') THEN 'VERSION'
ELSE 'NONE'
END AS pinnedStatus
END AS pinnedStatus,
(SELECT json_agg(json_build_object('file_size', file_size,
'hash', hash,
'file_name', file_name,
'external_url', external_url,
'platforms', platforms,
'download_platform', download_platform)) AS value
FROM project_version_downloads
WHERE version_id = pv.id
GROUP BY version_id) AS downloads
FROM project_versions pv
JOIN project_channels pc ON pv.channel_id = pc.id
<if(platformfilter)>INNER JOIN sq ON pv.id = sq.version_id<endif>

View File

@ -0,0 +1,56 @@
package io.papermc.hangar.db.mappers;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import io.papermc.hangar.model.api.project.version.FileInfo;
import io.papermc.hangar.model.api.project.version.PlatformVersionDownload;
import io.papermc.hangar.model.common.Platform;
import io.papermc.hangar.service.internal.file.FileService;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.EnumMap;
import java.util.Map;
import org.jdbi.v3.core.mapper.ColumnMapper;
import org.jdbi.v3.core.statement.StatementContext;
import org.springframework.stereotype.Component;
@Component
public class VersionDownloadsMapper implements ColumnMapper<Map<Platform, PlatformVersionDownload>> {
private final ObjectMapper objectMapper;
private final FileService fileService;
public VersionDownloadsMapper(final ObjectMapper objectMapper, final FileService fileService) {
this.objectMapper = objectMapper;
this.fileService = fileService;
}
@Override
public Map<Platform, PlatformVersionDownload> map(final ResultSet r, final int columnNumber, final StatementContext ctx) throws SQLException {
final String raw = r.getString(columnNumber);
final Map<Platform, PlatformVersionDownload> result = new EnumMap<>(Platform.class);
final String[] projectNamespace = (String[]) r.getArray("project_namespace").getArray();
final String version = r.getString("version_string");
try {
final JsonNode parsedDownloads = this.objectMapper.readTree(raw);
for (final JsonNode download : parsedDownloads) {
final JsonNode platforms = download.get("platforms");
final String fileName = download.get("file_name").asText();
final long fileSize = download.get("file_size").asLong();
final String hash = download.get("hash").asText();
final String externalUrl = download.get("external_url").asText();
final FileInfo fileInfo = "null".equals(fileName) ? null : new FileInfo(fileName, fileSize, hash);
final Platform downloadPlatform = Platform.values()[download.get("download_platform").asInt()];
final String downloadUrl = fileInfo != null ? this.fileService.getVersionDownloadUrl(projectNamespace[0], projectNamespace[1], version, downloadPlatform, fileInfo.getName()) : null;
final PlatformVersionDownload pvd = new PlatformVersionDownload(fileInfo, "null".equals(externalUrl) ? null : externalUrl, downloadUrl);
for (final JsonNode platformId : platforms) {
result.put(Platform.values()[platformId.asInt()], pvd);
}
}
} catch (JsonProcessingException e) {
throw new RuntimeException(e);
}
return result;
}
}

View File

@ -19,8 +19,19 @@ public class Version extends VersionCompact {
private final Map<Platform, Set<String>> platformDependencies = new EnumMap<>(Platform.class);
private final Map<Platform, List<String>> platformDependenciesFormatted = new EnumMap<>(Platform.class);
public Version(final OffsetDateTime createdAt, final long id, @ColumnName("version_string") final String name, final Visibility visibility, final String description, @Nested("vs") final VersionStats stats, final String author, @EnumByOrdinal final ReviewState reviewState, @Nested("pc") final ProjectChannel channel, final PinnedStatus pinnedStatus) {
super(createdAt, id, name, visibility, description, stats, author, reviewState, channel, pinnedStatus);
public Version(final OffsetDateTime createdAt,
final long id,
@ColumnName("version_string") final String name,
final Visibility visibility,
final String description,
@Nested("vs") final VersionStats stats,
final String author,
@EnumByOrdinal final ReviewState reviewState,
@Nested("pc") final ProjectChannel channel,
final PinnedStatus pinnedStatus,
final Map<Platform, PlatformVersionDownload> downloads
) {
super(createdAt, id, name, visibility, description, stats, author, reviewState, channel, pinnedStatus, downloads);
}
public Map<Platform, Set<PluginDependency>> getPluginDependencies() {

View File

@ -10,7 +10,6 @@ import io.papermc.hangar.model.common.Platform;
import io.papermc.hangar.model.common.projects.ReviewState;
import io.papermc.hangar.model.common.projects.Visibility;
import java.time.OffsetDateTime;
import java.util.EnumMap;
import java.util.Map;
import org.jdbi.v3.core.enums.EnumByName;
import org.jdbi.v3.core.enums.EnumByOrdinal;
@ -28,9 +27,19 @@ public class VersionCompact extends Model implements Named, Visible, Identified
private final ReviewState reviewState;
private final ProjectChannel channel;
private final PinnedStatus pinnedStatus;
private final Map<Platform, PlatformVersionDownload> downloads = new EnumMap<>(Platform.class);
private final Map<Platform, PlatformVersionDownload> downloads;
protected VersionCompact(final OffsetDateTime createdAt, final long id, @ColumnName("version_string") final String name, final Visibility visibility, final String description, @Nested("vs") final VersionStats stats, final String author, @EnumByOrdinal final ReviewState reviewState, @Nested("pc") final ProjectChannel channel, final PinnedStatus pinnedStatus) {
protected VersionCompact(final OffsetDateTime createdAt,
final long id,
@ColumnName("version_string") final String name,
final Visibility visibility,
final String description,
@Nested("vs") final VersionStats stats,
final String author,
@EnumByOrdinal final ReviewState reviewState,
@Nested("pc") final ProjectChannel channel,
final PinnedStatus pinnedStatus,
final Map<Platform, PlatformVersionDownload> downloads) {
super(createdAt);
this.id = id;
this.name = name;
@ -41,6 +50,7 @@ public class VersionCompact extends Model implements Named, Visible, Identified
this.reviewState = reviewState;
this.channel = channel;
this.pinnedStatus = pinnedStatus;
this.downloads = downloads;
}
@Override

View File

@ -111,6 +111,7 @@ public class SitemapService extends HangarComponent {
// add all versions of said projects
projects.forEach(p -> {
// TODO version compact, we dont need downloads here
final SortedMap<Long, Version> projectVersions = this.versionsApiDAO.getVersions(p.getId(), false, null, new RequestPagination(100L, 0L));
projectVersions.values().stream()
.filter(pv -> pv.getVisibility() == Visibility.PUBLIC || pv.getVisibility() == Visibility.NEEDSAPPROVAL)

View File

@ -44,6 +44,7 @@ import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.SortedMap;
import java.util.concurrent.CompletableFuture;
import java.util.function.Function;
import java.util.function.Supplier;
@ -172,9 +173,9 @@ public class ProjectService extends HangarComponent {
}
final Long userId = this.getHangarUserId();
final Long versionId = this.versionsApiDAO.getVersions(projectId, false, userId, pagination).keySet().stream().findAny().orElse(null);
if (versionId != null) {
return this.versionsApiDAO.getVersion(versionId, this.getGlobalPermissions().has(Permission.SeeHidden), userId);
final SortedMap<Long, Version> versions = this.versionsApiDAO.getVersions(projectId, this.getGlobalPermissions().has(Permission.SeeHidden), userId, pagination);
if (!versions.isEmpty()) {
return versions.values().iterator().next();
}
// Try again with any channel, else empty

View File

@ -3,8 +3,6 @@ package io.papermc.hangar.service.internal.versions;
import io.papermc.hangar.HangarComponent;
import io.papermc.hangar.db.dao.internal.table.versions.downloads.ProjectVersionDownloadsDAO;
import io.papermc.hangar.exceptions.HangarApiException;
import io.papermc.hangar.model.api.project.version.FileInfo;
import io.papermc.hangar.model.api.project.version.PlatformVersionDownload;
import io.papermc.hangar.model.common.Platform;
import io.papermc.hangar.model.db.projects.ProjectTable;
import io.papermc.hangar.model.db.versions.ProjectVersionTable;
@ -13,9 +11,6 @@ import io.papermc.hangar.service.internal.file.FileService;
import io.papermc.hangar.service.internal.file.S3FileService;
import io.papermc.hangar.service.internal.projects.ProjectService;
import io.papermc.hangar.service.internal.uploads.ProjectFiles;
import java.util.EnumMap;
import java.util.List;
import java.util.Map;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpStatus;
@ -39,19 +34,6 @@ public class DownloadService extends HangarComponent {
this.projectService = projectService;
}
public Map<Platform, PlatformVersionDownload> getDownloads(final String user, final String project, final String version, final long versionId) {
final Map<Platform, PlatformVersionDownload> versionDownloadsMap = new EnumMap<>(Platform.class);
final List<ProjectVersionDownloadTable> downloads = this.downloadsDAO.getDownloads(versionId);
for (final ProjectVersionDownloadTable download : downloads) {
final FileInfo fileInfo = download.getFileName() != null ? new FileInfo(download.getFileName(), download.getFileSize(), download.getHash()) : null;
final String downloadUrl = fileInfo != null ? this.fileService.getVersionDownloadUrl(user, project, version, download.getDownloadPlatform(), fileInfo.getName()) : null;
for (final Platform platform : download.getPlatforms()) {
versionDownloadsMap.put(platform, new PlatformVersionDownload(fileInfo, download.getExternalUrl(), downloadUrl));
}
}
return versionDownloadsMap;
}
public ResponseEntity<?> downloadVersion(final ProjectTable project, final ProjectVersionTable version, final Platform platform) {
final ProjectVersionDownloadTable download = this.downloadsDAO.getDownloadByPlatform(version.getVersionId(), platform);
if (download == null) {

View File

@ -2,20 +2,18 @@ package io.papermc.hangar.service.internal.versions;
import io.papermc.hangar.HangarComponent;
import io.papermc.hangar.db.customtypes.JSONB;
import io.papermc.hangar.db.dao.internal.table.projects.ProjectsDAO;
import io.papermc.hangar.db.dao.internal.table.versions.ProjectVersionReviewsDAO;
import io.papermc.hangar.db.dao.internal.table.versions.ProjectVersionsDAO;
import io.papermc.hangar.db.dao.internal.table.versions.downloads.ProjectVersionDownloadsDAO;
import io.papermc.hangar.db.dao.internal.versions.HangarReviewsDAO;
import io.papermc.hangar.exceptions.HangarApiException;
import io.papermc.hangar.model.api.project.version.PlatformVersionDownload;
import io.papermc.hangar.model.api.project.version.VersionToScan;
import io.papermc.hangar.model.common.Platform;
import io.papermc.hangar.model.common.ReviewAction;
import io.papermc.hangar.model.common.projects.ReviewState;
import io.papermc.hangar.model.common.projects.Visibility;
import io.papermc.hangar.model.db.UserTable;
import io.papermc.hangar.model.db.projects.ProjectTable;
import io.papermc.hangar.model.db.versions.ProjectVersionTable;
import io.papermc.hangar.model.db.versions.downloads.ProjectVersionDownloadTable;
import io.papermc.hangar.model.db.versions.reviews.ProjectVersionReviewMessageTable;
import io.papermc.hangar.model.db.versions.reviews.ProjectVersionReviewTable;
import io.papermc.hangar.model.internal.api.requests.versions.ReviewMessage;
@ -28,7 +26,6 @@ import io.papermc.hangar.service.internal.users.UserService;
import io.papermc.hangar.service.internal.visibility.ProjectVersionVisibilityService;
import java.time.OffsetDateTime;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpStatus;
@ -44,20 +41,18 @@ public class ReviewService extends HangarComponent {
private final ProjectVersionsDAO projectVersionsDAO;
private final ProjectVersionVisibilityService projectVersionVisibilityService;
private final NotificationService notificationService;
private final ProjectsDAO projectsDAO;
private final DownloadService downloadService;
private final UserService userService;
private final ProjectVersionDownloadsDAO downloadsDAO;
@Autowired
public ReviewService(final ProjectVersionReviewsDAO projectVersionReviewsDAO, final HangarReviewsDAO hangarReviewsDAO, final ProjectVersionsDAO projectVersionsDAO, final ProjectVersionVisibilityService projectVersionVisibilityService, final NotificationService notificationService, final ProjectsDAO projectsDAO, final DownloadService downloadService, final UserService userService) {
public ReviewService(final ProjectVersionReviewsDAO projectVersionReviewsDAO, final HangarReviewsDAO hangarReviewsDAO, final ProjectVersionsDAO projectVersionsDAO, final ProjectVersionVisibilityService projectVersionVisibilityService, final NotificationService notificationService, final UserService userService, final ProjectVersionDownloadsDAO downloadsDAO) {
this.projectVersionReviewsDAO = projectVersionReviewsDAO;
this.hangarReviewsDAO = hangarReviewsDAO;
this.projectVersionsDAO = projectVersionsDAO;
this.projectVersionVisibilityService = projectVersionVisibilityService;
this.notificationService = notificationService;
this.projectsDAO = projectsDAO;
this.downloadService = downloadService;
this.userService = userService;
this.downloadsDAO = downloadsDAO;
}
public List<HangarReview> getHangarReviews(final long versionId) {
@ -226,10 +221,8 @@ public class ReviewService extends HangarComponent {
final List<HangarReviewQueueEntry> reviewQueue = this.getReviewQueue(ReviewState.UNREVIEWED);
for (final HangarReviewQueueEntry entry : reviewQueue) {
final ProjectVersionTable projectVersionTable = this.projectVersionsDAO.getProjectVersionTable(entry.getVersionId());
final ProjectTable projectTable = this.projectsDAO.getById(projectVersionTable.getProjectId());
final Map<Platform, PlatformVersionDownload> downloads = this.downloadService.getDownloads(projectTable.getOwnerName(), projectTable.getSlug(),
projectVersionTable.getVersionString(), projectVersionTable.getVersionId());
if (downloads.values().stream().anyMatch(download -> download.externalUrl() == null || !this.config.security.checkSafe(download.externalUrl()))) {
final List<ProjectVersionDownloadTable> downloads = this.downloadsDAO.getDownloads(projectVersionTable.getVersionId());
if (downloads.stream().anyMatch(download -> download.getExternalUrl() == null || !this.config.security.checkSafe(download.getExternalUrl()))) {
continue;
}

View File

@ -7,7 +7,6 @@ import io.papermc.hangar.db.dao.internal.table.versions.dependencies.ProjectVers
import io.papermc.hangar.db.dao.internal.table.versions.dependencies.ProjectVersionPlatformDependenciesDAO;
import io.papermc.hangar.db.dao.v1.VersionsApiDAO;
import io.papermc.hangar.exceptions.HangarApiException;
import io.papermc.hangar.model.api.project.version.PlatformVersionDownload;
import io.papermc.hangar.model.api.project.version.PluginDependency;
import io.papermc.hangar.model.api.project.version.Version;
import io.papermc.hangar.model.common.Platform;
@ -45,16 +44,14 @@ public class VersionDependencyService extends HangarComponent {
private final ProjectsDAO projectsDAO;
private final ProjectVersionPlatformDependenciesDAO projectVersionPlatformDependenciesDAO;
private final PlatformService platformService;
private final DownloadService downloadService;
@Autowired
public VersionDependencyService(final ProjectVersionDependenciesDAO projectVersionDependencyDAO, final VersionsApiDAO versionsApiDAO, final ProjectsDAO projectsDAO, final ProjectVersionPlatformDependenciesDAO projectVersionPlatformDependencyDAO, final PlatformService platformService, final DownloadService downloadService) {
public VersionDependencyService(final ProjectVersionDependenciesDAO projectVersionDependencyDAO, final VersionsApiDAO versionsApiDAO, final ProjectsDAO projectsDAO, final ProjectVersionPlatformDependenciesDAO projectVersionPlatformDependencyDAO, final PlatformService platformService) {
this.projectVersionDependenciesDAO = projectVersionDependencyDAO;
this.versionsApiDAO = versionsApiDAO;
this.projectsDAO = projectsDAO;
this.projectVersionPlatformDependenciesDAO = projectVersionPlatformDependencyDAO;
this.platformService = platformService;
this.downloadService = downloadService;
}
@Cacheable(value = CacheConfig.VERSION_DEPENDENCIES, key = "#p3") // versionId is key
@ -72,26 +69,22 @@ public class VersionDependencyService extends HangarComponent {
final Map<Platform, SortedSet<PluginDependency>> pluginDependencies = this.versionsApiDAO.getPluginDependencies(versionId).stream()
.collect(Collectors.groupingBy(PluginDependency::getPlatform, Collectors.toCollection(TreeSet::new)));
final Map<Platform, PlatformVersionDownload> downloads = this.downloadService.getDownloads(user, project, versionName, versionId);
return new DownloadsAndDependencies(pluginDependencies, platformDependencies, platformDependenciesFormatted, downloads);
return new DownloadsAndDependencies(pluginDependencies, platformDependencies, platformDependenciesFormatted);
}
public record DownloadsAndDependencies(Map<Platform, SortedSet<PluginDependency>> pluginDependencies,
Map<Platform, SortedSet<String>> platformDependencies,
Map<Platform, List<String>> platformDependenciesFormatted,
Map<Platform, PlatformVersionDownload> downloads
Map<Platform, List<String>> platformDependenciesFormatted
) {
public <T extends Version> T applyTo(final T version) {
version.getPluginDependencies().putAll(this.pluginDependencies);
version.getPlatformDependencies().putAll(this.platformDependencies);
version.getPlatformDependenciesFormatted().putAll(this.platformDependenciesFormatted);
version.getDownloads().putAll(this.downloads);
return version;
}
public HangarProject.PinnedVersion applyTo(final HangarProject.PinnedVersion version) {
version.getPlatformDependenciesFormatted().putAll(this.platformDependenciesFormatted);
version.getDownloads().putAll(this.downloads);
return version;
}
}