feat: add total-sitemap.xml with all entries

backend/src/main/java/io/papermc/hangar/controller/SitemapController.java

@@ -25,6 +25,12 @@ public class SitemapController extends HangarComponent {
         return this.sitemapService.getSitemap();
     }
 
+    @Anyone
+    @GetMapping(value = "/total-sitemap.xml", produces = MediaType.APPLICATION_XML_VALUE)
+    public String totalSitemap() {
+        return this.sitemapService.getTotalSitemap();
+    }
+
     @Anyone
     @GetMapping(value = "/global-sitemap.xml", produces = MediaType.APPLICATION_XML_VALUE)
     public String globalSitemap() {

backend/src/main/java/io/papermc/hangar/db/dao/internal/table/UserDAO.java

@@ -74,4 +74,7 @@ public interface UserDAO {
     @Timestamped
     @SqlUpdate("INSERT INTO users_history(uuid, old_name, new_name, date) VALUES (:uuid, :oldName, :newName, :now)")
     void recordNameChange(final @NotNull UUID uuid, final @NotNull String oldName, final @NotNull String newName);
+
+    @SqlQuery("SELECT * FROM users")
+    List<UserTable> getUsers();
 }

backend/src/main/java/io/papermc/hangar/service/internal/SitemapService.java

@@ -1,6 +1,5 @@
 package io.papermc.hangar.service.internal;
 
-import cz.jiripinkas.jsitemapgenerator.ChangeFreq;
 import cz.jiripinkas.jsitemapgenerator.WebPage;
 import cz.jiripinkas.jsitemapgenerator.generator.SitemapGenerator;
 import cz.jiripinkas.jsitemapgenerator.generator.SitemapIndexGenerator;
@@ -9,32 +8,61 @@ import io.papermc.hangar.config.CacheConfig;
 import io.papermc.hangar.db.dao.internal.projects.HangarProjectPagesDAO;
 import io.papermc.hangar.db.dao.internal.table.UserDAO;
 import io.papermc.hangar.db.dao.internal.table.projects.ProjectsDAO;
-import io.papermc.hangar.db.dao.internal.table.versions.ProjectVersionsDAO;
+import io.papermc.hangar.db.dao.v1.VersionsApiDAO;
 import io.papermc.hangar.exceptions.HangarApiException;
+import io.papermc.hangar.model.api.project.version.Version;
+import io.papermc.hangar.model.api.requests.RequestPagination;
 import io.papermc.hangar.model.common.projects.Visibility;
 import io.papermc.hangar.model.db.UserTable;
 import io.papermc.hangar.model.db.projects.ProjectTable;
-import io.papermc.hangar.model.db.versions.ProjectVersionTable;
 import io.papermc.hangar.model.internal.projects.ExtendedProjectPage;
+import java.time.Duration;
+import java.time.LocalDateTime;
 import java.util.List;
+import java.util.SortedMap;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.springframework.cache.annotation.Cacheable;
+import org.springframework.scheduling.annotation.Scheduled;
 import org.springframework.stereotype.Service;
 
 @Service
 public class SitemapService extends HangarComponent {
 
+    private static final Logger logger = LoggerFactory.getLogger(SitemapService.class);
+
     private final UserDAO userDAO;
     private final ProjectsDAO projectsDAO;
-    private final ProjectVersionsDAO projectVersionsDAO;
+    private final VersionsApiDAO versionsApiDAO;
     private final HangarProjectPagesDAO hangarProjectPagesDAO;
 
-    public SitemapService(final UserDAO userDAO, final ProjectsDAO projectsDAO, final ProjectVersionsDAO projectVersionsDAO, final HangarProjectPagesDAO hangarProjectPagesDAO) {
+    private String totalSitemap;
+
+    public SitemapService(final UserDAO userDAO, final ProjectsDAO projectsDAO, final VersionsApiDAO versionsApiDAO, final HangarProjectPagesDAO hangarProjectPagesDAO) {
         this.userDAO = userDAO;
         this.projectsDAO = projectsDAO;
-        this.projectVersionsDAO = projectVersionsDAO;
+        this.versionsApiDAO = versionsApiDAO;
         this.hangarProjectPagesDAO = hangarProjectPagesDAO;
     }
 
+    @Scheduled(fixedDelayString = "PT6H", initialDelayString = "PT1S")
+    public void updateTotalSitemap() {
+        logger.info("Updating sitemap...");
+        LocalDateTime start = LocalDateTime.now();
+        SitemapGenerator generator = SitemapGenerator.of(this.config.getBaseUrl());
+        addGlobal(generator);
+        this.userDAO.getUsers().forEach((u) -> addUser(u, generator));
+        this.totalSitemap = generator.toString();
+        logger.info("Updated sitemap, took {}.", Duration.between(start, LocalDateTime.now()));
+    }
+
+    public String getTotalSitemap() {
+        if (this.totalSitemap == null) {
+            throw HangarApiException.notFound();
+        }
+        return this.totalSitemap;
+    }
+
     @Cacheable(CacheConfig.INDEX_SITEMAP)
     public String getSitemap() {
         final SitemapIndexGenerator generator = SitemapIndexGenerator.of(this.config.getBaseUrl())
@@ -46,19 +74,23 @@ public class SitemapService extends HangarComponent {
 
     @Cacheable(CacheConfig.GLOBAL_SITEMAP)
     public String getGlobalSitemap() {
-        return SitemapGenerator.of(this.config.getBaseUrl())
+        return addGlobal(SitemapGenerator.of(this.config.getBaseUrl())).toString();
-            .addPage(WebPage.builder().name("").changeFreq(ChangeFreq.HOURLY).build())
+    }
-            .addPage(WebPage.builder().name("paper").changeFreq(ChangeFreq.HOURLY).build())
+
-            .addPage(WebPage.builder().name("waterfall").changeFreq(ChangeFreq.HOURLY).build())
+    private static SitemapGenerator addGlobal(SitemapGenerator generator) {
-            .addPage(WebPage.builder().name("velocity").changeFreq(ChangeFreq.HOURLY).build())
+        generator
-            .addPage(WebPage.builder().name("authors").changeFreq(ChangeFreq.WEEKLY).build())
+            .addPage(WebPage.builder().name("").changeFreqDaily().priority(1.0).build())
-            .addPage(WebPage.builder().name("staff").changeFreq(ChangeFreq.WEEKLY).build())
+            .addPage(WebPage.builder().name("paper").changeFreqDaily().priority(1.0).build())
-            .addPage(WebPage.builder().name("guidelines").changeFreq(ChangeFreq.WEEKLY).build())
+            .addPage(WebPage.builder().name("waterfall").changeFreqDaily().priority(1.0).build())
-            .addPage(WebPage.builder().name("terms").changeFreq(ChangeFreq.WEEKLY).build())
+            .addPage(WebPage.builder().name("velocity").changeFreqDaily().priority(1.0).build())
-            .addPage(WebPage.builder().name("privacy").changeFreq(ChangeFreq.WEEKLY).build())
+            .addPage(WebPage.builder().name("authors").changeFreqWeekly().priority(1.0).build())
-            .addPage(WebPage.builder().name("version").changeFreq(ChangeFreq.DAILY).build())
+            .addPage(WebPage.builder().name("staff").changeFreqWeekly().priority(1.0).build())
-            .addPage(WebPage.builder().name("api-docs").build())
+            .addPage(WebPage.builder().name("guidelines").changeFreqMonthly().priority(1.0).build())
-            .toString();
+            .addPage(WebPage.builder().name("terms").changeFreqMonthly().priority(1.0).build())
+            .addPage(WebPage.builder().name("privacy").changeFreqMonthly().priority(1.0).build())
+            .addPage(WebPage.builder().name("version").changeFreqWeekly().priority(1.0).build())
+            .addPage(WebPage.builder().name("api-docs").changeFreqMonthly().priority(1.0).build());
+        return generator;
     }
 
     @Cacheable(value = CacheConfig.USER_SITEMAP, key = "#username")
@@ -68,20 +100,25 @@ public class SitemapService extends HangarComponent {
             throw HangarApiException.notFound();
         }
 
-        final SitemapGenerator generator = SitemapGenerator.of(this.config.getBaseUrl());
+        return addUser(userTable, SitemapGenerator.of(this.config.getBaseUrl())).toString();
-        generator.defaultChangeFreqWeekly();
+    }
 
+    private SitemapGenerator addUser(UserTable userTable, SitemapGenerator generator) {
         // add all projects
         final List<ProjectTable> projects = this.projectsDAO.getUserProjects(userTable.getId(), false);
         projects.removeIf(p -> p.getVisibility() != Visibility.PUBLIC && p.getVisibility() != Visibility.NEEDSAPPROVAL);
-        projects.forEach(p -> generator.addPage(userTable.getName() + "/" + p.getSlug()));
+        projects.forEach(p -> generator.addPage(WebPage.builder().name(userTable.getName() + "/" + p.getSlug()).changeFreqWeekly().priority(0.9).build()));
 
         // add all versions of said projects
         projects.forEach(p -> {
-            final List<ProjectVersionTable> projectVersions = this.projectVersionsDAO.getProjectVersions(p.getId());
+            final SortedMap<Long, Version> projectVersions = this.versionsApiDAO.getVersions(p.getSlug(), false, null, new RequestPagination(100L, 0L));
-            projectVersions.stream()
+            projectVersions.values().stream()
                 .filter(pv -> pv.getVisibility() == Visibility.PUBLIC || pv.getVisibility() == Visibility.NEEDSAPPROVAL)
-                .forEach(pv -> generator.addPage(this.path(userTable.getName(), p.getSlug(), "versions", pv.getVersionString())));
+                .forEach(pv -> generator.addPage(WebPage.builder().name(this.path(userTable.getName(), p.getSlug(), "versions", pv.getName()))
+                    .priority(pv.getChannel().getName().equals(this.config.channels.nameDefault()) ? 0.5 : 0.1)
+                    .changeFreqMonthly()
+                    .build())
+                );
         });
 
         // add all pages of said projects
@@ -91,14 +128,14 @@ public class SitemapService extends HangarComponent {
                 if (pp.isHome()) {
                     continue;
                 }
-                generator.addPage(this.path(userTable.getName(), project.getSlug(), "pages", pp.getSlug()));
+                generator.addPage(WebPage.builder().name(this.path(userTable.getName(), project.getSlug(), "pages", pp.getSlug())).priority(0.5).changeFreqWeekly().build());
             }
         });
 
         // lastly, add user page
-        generator.addPage(WebPage.builder().name(userTable.getName()).build());
+        generator.addPage(WebPage.builder().name(userTable.getName()).priority(projects.isEmpty() ? 0.1 : 0.4).changeFreqWeekly().build());
 
-        return generator.toString();
+        return generator;
     }
 
     private String path(final String... paths) {

backend/src/main/java/io/papermc/hangar/service/internal/versions/JarScanningService.java

@@ -27,6 +27,8 @@ import io.papermc.hangar.util.ThreadFactory;
 import io.sentry.Sentry;
 import java.io.IOException;
 import java.io.InputStream;
+import java.time.Duration;
+import java.time.LocalDateTime;
 import java.util.Comparator;
 import java.util.List;
 import java.util.Locale;
@@ -86,10 +88,11 @@ public class JarScanningService {
         withTransaction("task", "JarScanningService#scanRemainingProjectVersions", () -> {
             // TODO Pass this.scanner.version()
             final List<VersionToScan> versionToScans = this.dao.versionsRequiringScans();
+            LocalDateTime start = LocalDateTime.now();
             LOGGER.info("Rescanning {} versions", versionToScans.size());
-            for (final VersionToScan version : versionToScans) {
+            // TODO partial parameter
-                this.scan(version, false); // TODO partial parameter
+            versionToScans.forEach(version -> this.scan(version, false));
-            }
+            LOGGER.info("Rescanned. Took {}.", Duration.between(start, LocalDateTime.now()));
         });
     }
 

chart/values.yaml

@@ -39,6 +39,9 @@ ingress:
     - path: /sitemap.xml
       pathType: ImplementationSpecific
       target: backend
+    - path: /total-sitemap.xml
+      pathType: ImplementationSpecific
+      target: backend
     - path: /global-sitemap.xml
       pathType: ImplementationSpecific
       target: backend