From 6e143b4d7abcec999c52831d3f949532fc65edc7 Mon Sep 17 00:00:00 2001 From: MiniDigger | Martin Date: Sun, 17 Nov 2024 10:47:46 +0100 Subject: [PATCH] feat: add total-sitemap.xml with all entries --- .../hangar/controller/SitemapController.java | 6 ++ .../hangar/db/dao/internal/table/UserDAO.java | 3 + .../service/internal/SitemapService.java | 93 +++++++++++++------ .../internal/versions/JarScanningService.java | 9 +- chart/values.yaml | 3 + 5 files changed, 83 insertions(+), 31 deletions(-) diff --git a/backend/src/main/java/io/papermc/hangar/controller/SitemapController.java b/backend/src/main/java/io/papermc/hangar/controller/SitemapController.java index a193681b..0c5770b4 100644 --- a/backend/src/main/java/io/papermc/hangar/controller/SitemapController.java +++ b/backend/src/main/java/io/papermc/hangar/controller/SitemapController.java @@ -25,6 +25,12 @@ public class SitemapController extends HangarComponent { return this.sitemapService.getSitemap(); } + @Anyone + @GetMapping(value = "/total-sitemap.xml", produces = MediaType.APPLICATION_XML_VALUE) + public String totalSitemap() { + return this.sitemapService.getTotalSitemap(); + } + @Anyone @GetMapping(value = "/global-sitemap.xml", produces = MediaType.APPLICATION_XML_VALUE) public String globalSitemap() { diff --git a/backend/src/main/java/io/papermc/hangar/db/dao/internal/table/UserDAO.java b/backend/src/main/java/io/papermc/hangar/db/dao/internal/table/UserDAO.java index b470dd87..39ba85f3 100644 --- a/backend/src/main/java/io/papermc/hangar/db/dao/internal/table/UserDAO.java +++ b/backend/src/main/java/io/papermc/hangar/db/dao/internal/table/UserDAO.java @@ -74,4 +74,7 @@ public interface UserDAO { @Timestamped @SqlUpdate("INSERT INTO users_history(uuid, old_name, new_name, date) VALUES (:uuid, :oldName, :newName, :now)") void recordNameChange(final @NotNull UUID uuid, final @NotNull String oldName, final @NotNull String newName); + + @SqlQuery("SELECT * FROM users") + List getUsers(); } diff --git a/backend/src/main/java/io/papermc/hangar/service/internal/SitemapService.java b/backend/src/main/java/io/papermc/hangar/service/internal/SitemapService.java index a332eff4..3f854fbf 100644 --- a/backend/src/main/java/io/papermc/hangar/service/internal/SitemapService.java +++ b/backend/src/main/java/io/papermc/hangar/service/internal/SitemapService.java @@ -1,6 +1,5 @@ package io.papermc.hangar.service.internal; -import cz.jiripinkas.jsitemapgenerator.ChangeFreq; import cz.jiripinkas.jsitemapgenerator.WebPage; import cz.jiripinkas.jsitemapgenerator.generator.SitemapGenerator; import cz.jiripinkas.jsitemapgenerator.generator.SitemapIndexGenerator; @@ -9,32 +8,61 @@ import io.papermc.hangar.config.CacheConfig; import io.papermc.hangar.db.dao.internal.projects.HangarProjectPagesDAO; import io.papermc.hangar.db.dao.internal.table.UserDAO; import io.papermc.hangar.db.dao.internal.table.projects.ProjectsDAO; -import io.papermc.hangar.db.dao.internal.table.versions.ProjectVersionsDAO; +import io.papermc.hangar.db.dao.v1.VersionsApiDAO; import io.papermc.hangar.exceptions.HangarApiException; +import io.papermc.hangar.model.api.project.version.Version; +import io.papermc.hangar.model.api.requests.RequestPagination; import io.papermc.hangar.model.common.projects.Visibility; import io.papermc.hangar.model.db.UserTable; import io.papermc.hangar.model.db.projects.ProjectTable; -import io.papermc.hangar.model.db.versions.ProjectVersionTable; import io.papermc.hangar.model.internal.projects.ExtendedProjectPage; +import java.time.Duration; +import java.time.LocalDateTime; import java.util.List; +import java.util.SortedMap; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.cache.annotation.Cacheable; +import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; @Service public class SitemapService extends HangarComponent { + private static final Logger logger = LoggerFactory.getLogger(SitemapService.class); + private final UserDAO userDAO; private final ProjectsDAO projectsDAO; - private final ProjectVersionsDAO projectVersionsDAO; + private final VersionsApiDAO versionsApiDAO; private final HangarProjectPagesDAO hangarProjectPagesDAO; - public SitemapService(final UserDAO userDAO, final ProjectsDAO projectsDAO, final ProjectVersionsDAO projectVersionsDAO, final HangarProjectPagesDAO hangarProjectPagesDAO) { + private String totalSitemap; + + public SitemapService(final UserDAO userDAO, final ProjectsDAO projectsDAO, final VersionsApiDAO versionsApiDAO, final HangarProjectPagesDAO hangarProjectPagesDAO) { this.userDAO = userDAO; this.projectsDAO = projectsDAO; - this.projectVersionsDAO = projectVersionsDAO; + this.versionsApiDAO = versionsApiDAO; this.hangarProjectPagesDAO = hangarProjectPagesDAO; } + @Scheduled(fixedDelayString = "PT6H", initialDelayString = "PT1S") + public void updateTotalSitemap() { + logger.info("Updating sitemap..."); + LocalDateTime start = LocalDateTime.now(); + SitemapGenerator generator = SitemapGenerator.of(this.config.getBaseUrl()); + addGlobal(generator); + this.userDAO.getUsers().forEach((u) -> addUser(u, generator)); + this.totalSitemap = generator.toString(); + logger.info("Updated sitemap, took {}.", Duration.between(start, LocalDateTime.now())); + } + + public String getTotalSitemap() { + if (this.totalSitemap == null) { + throw HangarApiException.notFound(); + } + return this.totalSitemap; + } + @Cacheable(CacheConfig.INDEX_SITEMAP) public String getSitemap() { final SitemapIndexGenerator generator = SitemapIndexGenerator.of(this.config.getBaseUrl()) @@ -46,19 +74,23 @@ public class SitemapService extends HangarComponent { @Cacheable(CacheConfig.GLOBAL_SITEMAP) public String getGlobalSitemap() { - return SitemapGenerator.of(this.config.getBaseUrl()) - .addPage(WebPage.builder().name("").changeFreq(ChangeFreq.HOURLY).build()) - .addPage(WebPage.builder().name("paper").changeFreq(ChangeFreq.HOURLY).build()) - .addPage(WebPage.builder().name("waterfall").changeFreq(ChangeFreq.HOURLY).build()) - .addPage(WebPage.builder().name("velocity").changeFreq(ChangeFreq.HOURLY).build()) - .addPage(WebPage.builder().name("authors").changeFreq(ChangeFreq.WEEKLY).build()) - .addPage(WebPage.builder().name("staff").changeFreq(ChangeFreq.WEEKLY).build()) - .addPage(WebPage.builder().name("guidelines").changeFreq(ChangeFreq.WEEKLY).build()) - .addPage(WebPage.builder().name("terms").changeFreq(ChangeFreq.WEEKLY).build()) - .addPage(WebPage.builder().name("privacy").changeFreq(ChangeFreq.WEEKLY).build()) - .addPage(WebPage.builder().name("version").changeFreq(ChangeFreq.DAILY).build()) - .addPage(WebPage.builder().name("api-docs").build()) - .toString(); + return addGlobal(SitemapGenerator.of(this.config.getBaseUrl())).toString(); + } + + private static SitemapGenerator addGlobal(SitemapGenerator generator) { + generator + .addPage(WebPage.builder().name("").changeFreqDaily().priority(1.0).build()) + .addPage(WebPage.builder().name("paper").changeFreqDaily().priority(1.0).build()) + .addPage(WebPage.builder().name("waterfall").changeFreqDaily().priority(1.0).build()) + .addPage(WebPage.builder().name("velocity").changeFreqDaily().priority(1.0).build()) + .addPage(WebPage.builder().name("authors").changeFreqWeekly().priority(1.0).build()) + .addPage(WebPage.builder().name("staff").changeFreqWeekly().priority(1.0).build()) + .addPage(WebPage.builder().name("guidelines").changeFreqMonthly().priority(1.0).build()) + .addPage(WebPage.builder().name("terms").changeFreqMonthly().priority(1.0).build()) + .addPage(WebPage.builder().name("privacy").changeFreqMonthly().priority(1.0).build()) + .addPage(WebPage.builder().name("version").changeFreqWeekly().priority(1.0).build()) + .addPage(WebPage.builder().name("api-docs").changeFreqMonthly().priority(1.0).build()); + return generator; } @Cacheable(value = CacheConfig.USER_SITEMAP, key = "#username") @@ -68,20 +100,25 @@ public class SitemapService extends HangarComponent { throw HangarApiException.notFound(); } - final SitemapGenerator generator = SitemapGenerator.of(this.config.getBaseUrl()); - generator.defaultChangeFreqWeekly(); + return addUser(userTable, SitemapGenerator.of(this.config.getBaseUrl())).toString(); + } + private SitemapGenerator addUser(UserTable userTable, SitemapGenerator generator) { // add all projects final List projects = this.projectsDAO.getUserProjects(userTable.getId(), false); projects.removeIf(p -> p.getVisibility() != Visibility.PUBLIC && p.getVisibility() != Visibility.NEEDSAPPROVAL); - projects.forEach(p -> generator.addPage(userTable.getName() + "/" + p.getSlug())); + projects.forEach(p -> generator.addPage(WebPage.builder().name(userTable.getName() + "/" + p.getSlug()).changeFreqWeekly().priority(0.9).build())); // add all versions of said projects projects.forEach(p -> { - final List projectVersions = this.projectVersionsDAO.getProjectVersions(p.getId()); - projectVersions.stream() + final SortedMap projectVersions = this.versionsApiDAO.getVersions(p.getSlug(), false, null, new RequestPagination(100L, 0L)); + projectVersions.values().stream() .filter(pv -> pv.getVisibility() == Visibility.PUBLIC || pv.getVisibility() == Visibility.NEEDSAPPROVAL) - .forEach(pv -> generator.addPage(this.path(userTable.getName(), p.getSlug(), "versions", pv.getVersionString()))); + .forEach(pv -> generator.addPage(WebPage.builder().name(this.path(userTable.getName(), p.getSlug(), "versions", pv.getName())) + .priority(pv.getChannel().getName().equals(this.config.channels.nameDefault()) ? 0.5 : 0.1) + .changeFreqMonthly() + .build()) + ); }); // add all pages of said projects @@ -91,14 +128,14 @@ public class SitemapService extends HangarComponent { if (pp.isHome()) { continue; } - generator.addPage(this.path(userTable.getName(), project.getSlug(), "pages", pp.getSlug())); + generator.addPage(WebPage.builder().name(this.path(userTable.getName(), project.getSlug(), "pages", pp.getSlug())).priority(0.5).changeFreqWeekly().build()); } }); // lastly, add user page - generator.addPage(WebPage.builder().name(userTable.getName()).build()); + generator.addPage(WebPage.builder().name(userTable.getName()).priority(projects.isEmpty() ? 0.1 : 0.4).changeFreqWeekly().build()); - return generator.toString(); + return generator; } private String path(final String... paths) { diff --git a/backend/src/main/java/io/papermc/hangar/service/internal/versions/JarScanningService.java b/backend/src/main/java/io/papermc/hangar/service/internal/versions/JarScanningService.java index a86de3e2..b33ac2f5 100644 --- a/backend/src/main/java/io/papermc/hangar/service/internal/versions/JarScanningService.java +++ b/backend/src/main/java/io/papermc/hangar/service/internal/versions/JarScanningService.java @@ -27,6 +27,8 @@ import io.papermc.hangar.util.ThreadFactory; import io.sentry.Sentry; import java.io.IOException; import java.io.InputStream; +import java.time.Duration; +import java.time.LocalDateTime; import java.util.Comparator; import java.util.List; import java.util.Locale; @@ -86,10 +88,11 @@ public class JarScanningService { withTransaction("task", "JarScanningService#scanRemainingProjectVersions", () -> { // TODO Pass this.scanner.version() final List versionToScans = this.dao.versionsRequiringScans(); + LocalDateTime start = LocalDateTime.now(); LOGGER.info("Rescanning {} versions", versionToScans.size()); - for (final VersionToScan version : versionToScans) { - this.scan(version, false); // TODO partial parameter - } + // TODO partial parameter + versionToScans.forEach(version -> this.scan(version, false)); + LOGGER.info("Rescanned. Took {}.", Duration.between(start, LocalDateTime.now())); }); } diff --git a/chart/values.yaml b/chart/values.yaml index 4ae343df..05c79100 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -39,6 +39,9 @@ ingress: - path: /sitemap.xml pathType: ImplementationSpecific target: backend + - path: /total-sitemap.xml + pathType: ImplementationSpecific + target: backend - path: /global-sitemap.xml pathType: ImplementationSpecific target: backend