mirror/Hangar
2
0
Fork 0
mirror of https://github.com/HangarMC/Hangar.git synced 2024-11-21 01:21:54 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: add total-sitemap.xml with all entries

Browse Source
This commit is contained in:
MiniDigger | Martin 2024-11-17 10:47:46 +01:00
parent 723c511337
commit 6e143b4d7a
5 changed files with 83 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
backend/src/main/java/io/papermc/hangar/controller/SitemapController.java
View File

@ -25,6 +25,12 @@ public class SitemapController extends HangarComponent {
return this.sitemapService.getSitemap();
}
@Anyone
@GetMapping(value = "/total-sitemap.xml", produces = MediaType.APPLICATION_XML_VALUE)
public String totalSitemap() {
return this.sitemapService.getTotalSitemap();
}
@Anyone
@GetMapping(value = "/global-sitemap.xml", produces = MediaType.APPLICATION_XML_VALUE)
public String globalSitemap() {

3
backend/src/main/java/io/papermc/hangar/db/dao/internal/table/UserDAO.java
View File

@ -74,4 +74,7 @@ public interface UserDAO {
@Timestamped
@SqlUpdate("INSERT INTO users_history(uuid, old_name, new_name, date) VALUES (:uuid, :oldName, :newName, :now)")
void recordNameChange(final @NotNull UUID uuid, final @NotNull String oldName, final @NotNull String newName);
@SqlQuery("SELECT * FROM users")
List<UserTable> getUsers();
}

93
backend/src/main/java/io/papermc/hangar/service/internal/SitemapService.java
View File

@ -1,6 +1,5 @@
package io.papermc.hangar.service.internal;
import cz.jiripinkas.jsitemapgenerator.ChangeFreq;
import cz.jiripinkas.jsitemapgenerator.WebPage;
import cz.jiripinkas.jsitemapgenerator.generator.SitemapGenerator;
import cz.jiripinkas.jsitemapgenerator.generator.SitemapIndexGenerator;
@ -9,32 +8,61 @@ import io.papermc.hangar.config.CacheConfig;
import io.papermc.hangar.db.dao.internal.projects.HangarProjectPagesDAO;
import io.papermc.hangar.db.dao.internal.table.UserDAO;
import io.papermc.hangar.db.dao.internal.table.projects.ProjectsDAO;
import io.papermc.hangar.db.dao.internal.table.versions.ProjectVersionsDAO;
import io.papermc.hangar.db.dao.v1.VersionsApiDAO;
import io.papermc.hangar.exceptions.HangarApiException;
import io.papermc.hangar.model.api.project.version.Version;
import io.papermc.hangar.model.api.requests.RequestPagination;
import io.papermc.hangar.model.common.projects.Visibility;
import io.papermc.hangar.model.db.UserTable;
import io.papermc.hangar.model.db.projects.ProjectTable;
import io.papermc.hangar.model.db.versions.ProjectVersionTable;
import io.papermc.hangar.model.internal.projects.ExtendedProjectPage;
import java.time.Duration;
import java.time.LocalDateTime;
import java.util.List;
import java.util.SortedMap;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.cache.annotation.Cacheable;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
@Service
public class SitemapService extends HangarComponent {
private static final Logger logger = LoggerFactory.getLogger(SitemapService.class);
private final UserDAO userDAO;
private final ProjectsDAO projectsDAO;
private final ProjectVersionsDAO projectVersionsDAO;
private final VersionsApiDAO versionsApiDAO;
private final HangarProjectPagesDAO hangarProjectPagesDAO;
public SitemapService(final UserDAO userDAO, final ProjectsDAO projectsDAO, final ProjectVersionsDAO projectVersionsDAO, final HangarProjectPagesDAO hangarProjectPagesDAO) {
private String totalSitemap;
public SitemapService(final UserDAO userDAO, final ProjectsDAO projectsDAO, final VersionsApiDAO versionsApiDAO, final HangarProjectPagesDAO hangarProjectPagesDAO) {
this.userDAO = userDAO;
this.projectsDAO = projectsDAO;
this.projectVersionsDAO = projectVersionsDAO;
this.versionsApiDAO = versionsApiDAO;
this.hangarProjectPagesDAO = hangarProjectPagesDAO;
}
@Scheduled(fixedDelayString = "PT6H", initialDelayString = "PT1S")
public void updateTotalSitemap() {
logger.info("Updating sitemap...");
LocalDateTime start = LocalDateTime.now();
SitemapGenerator generator = SitemapGenerator.of(this.config.getBaseUrl());
addGlobal(generator);
this.userDAO.getUsers().forEach((u) -> addUser(u, generator));
this.totalSitemap = generator.toString();
logger.info("Updated sitemap, took {}.", Duration.between(start, LocalDateTime.now()));
}
public String getTotalSitemap() {
if (this.totalSitemap == null) {
throw HangarApiException.notFound();
}
return this.totalSitemap;
}
@Cacheable(CacheConfig.INDEX_SITEMAP)
public String getSitemap() {
final SitemapIndexGenerator generator = SitemapIndexGenerator.of(this.config.getBaseUrl())
@ -46,19 +74,23 @@ public class SitemapService extends HangarComponent {
@Cacheable(CacheConfig.GLOBAL_SITEMAP)
public String getGlobalSitemap() {
return SitemapGenerator.of(this.config.getBaseUrl())
.addPage(WebPage.builder().name("").changeFreq(ChangeFreq.HOURLY).build())
.addPage(WebPage.builder().name("paper").changeFreq(ChangeFreq.HOURLY).build())
.addPage(WebPage.builder().name("waterfall").changeFreq(ChangeFreq.HOURLY).build())
.addPage(WebPage.builder().name("velocity").changeFreq(ChangeFreq.HOURLY).build())
.addPage(WebPage.builder().name("authors").changeFreq(ChangeFreq.WEEKLY).build())
.addPage(WebPage.builder().name("staff").changeFreq(ChangeFreq.WEEKLY).build())
.addPage(WebPage.builder().name("guidelines").changeFreq(ChangeFreq.WEEKLY).build())
.addPage(WebPage.builder().name("terms").changeFreq(ChangeFreq.WEEKLY).build())
.addPage(WebPage.builder().name("privacy").changeFreq(ChangeFreq.WEEKLY).build())
.addPage(WebPage.builder().name("version").changeFreq(ChangeFreq.DAILY).build())
.addPage(WebPage.builder().name("api-docs").build())
.toString();
return addGlobal(SitemapGenerator.of(this.config.getBaseUrl())).toString();
}
private static SitemapGenerator addGlobal(SitemapGenerator generator) {
generator
.addPage(WebPage.builder().name("").changeFreqDaily().priority(1.0).build())
.addPage(WebPage.builder().name("paper").changeFreqDaily().priority(1.0).build())
.addPage(WebPage.builder().name("waterfall").changeFreqDaily().priority(1.0).build())
.addPage(WebPage.builder().name("velocity").changeFreqDaily().priority(1.0).build())
.addPage(WebPage.builder().name("authors").changeFreqWeekly().priority(1.0).build())
.addPage(WebPage.builder().name("staff").changeFreqWeekly().priority(1.0).build())
.addPage(WebPage.builder().name("guidelines").changeFreqMonthly().priority(1.0).build())
.addPage(WebPage.builder().name("terms").changeFreqMonthly().priority(1.0).build())
.addPage(WebPage.builder().name("privacy").changeFreqMonthly().priority(1.0).build())
.addPage(WebPage.builder().name("version").changeFreqWeekly().priority(1.0).build())
.addPage(WebPage.builder().name("api-docs").changeFreqMonthly().priority(1.0).build());
return generator;
}
@Cacheable(value = CacheConfig.USER_SITEMAP, key = "#username")
@ -68,20 +100,25 @@ public class SitemapService extends HangarComponent {
throw HangarApiException.notFound();
}
final SitemapGenerator generator = SitemapGenerator.of(this.config.getBaseUrl());
generator.defaultChangeFreqWeekly();
return addUser(userTable, SitemapGenerator.of(this.config.getBaseUrl())).toString();
}
private SitemapGenerator addUser(UserTable userTable, SitemapGenerator generator) {
// add all projects
final List<ProjectTable> projects = this.projectsDAO.getUserProjects(userTable.getId(), false);
projects.removeIf(p -> p.getVisibility() != Visibility.PUBLIC && p.getVisibility() != Visibility.NEEDSAPPROVAL);
projects.forEach(p -> generator.addPage(userTable.getName() + "/" + p.getSlug()));
projects.forEach(p -> generator.addPage(WebPage.builder().name(userTable.getName() + "/" + p.getSlug()).changeFreqWeekly().priority(0.9).build()));
// add all versions of said projects
projects.forEach(p -> {
final List<ProjectVersionTable> projectVersions = this.projectVersionsDAO.getProjectVersions(p.getId());
projectVersions.stream()
final SortedMap<Long, Version> projectVersions = this.versionsApiDAO.getVersions(p.getSlug(), false, null, new RequestPagination(100L, 0L));
projectVersions.values().stream()
.filter(pv -> pv.getVisibility() == Visibility.PUBLIC || pv.getVisibility() == Visibility.NEEDSAPPROVAL)
.forEach(pv -> generator.addPage(this.path(userTable.getName(), p.getSlug(), "versions", pv.getVersionString())));
.forEach(pv -> generator.addPage(WebPage.builder().name(this.path(userTable.getName(), p.getSlug(), "versions", pv.getName()))
.priority(pv.getChannel().getName().equals(this.config.channels.nameDefault()) ? 0.5 : 0.1)
.changeFreqMonthly()
.build())
);
});
// add all pages of said projects
@ -91,14 +128,14 @@ public class SitemapService extends HangarComponent {
if (pp.isHome()) {
continue;
}
generator.addPage(this.path(userTable.getName(), project.getSlug(), "pages", pp.getSlug()));
generator.addPage(WebPage.builder().name(this.path(userTable.getName(), project.getSlug(), "pages", pp.getSlug())).priority(0.5).changeFreqWeekly().build());
}
});
// lastly, add user page
generator.addPage(WebPage.builder().name(userTable.getName()).build());
generator.addPage(WebPage.builder().name(userTable.getName()).priority(projects.isEmpty() ? 0.1 : 0.4).changeFreqWeekly().build());
return generator.toString();
return generator;
}
private String path(final String... paths) {

9
backend/src/main/java/io/papermc/hangar/service/internal/versions/JarScanningService.java
View File

@ -27,6 +27,8 @@ import io.papermc.hangar.util.ThreadFactory;
import io.sentry.Sentry;
import java.io.IOException;
import java.io.InputStream;
import java.time.Duration;
import java.time.LocalDateTime;
import java.util.Comparator;
import java.util.List;
import java.util.Locale;
@ -86,10 +88,11 @@ public class JarScanningService {
withTransaction("task", "JarScanningService#scanRemainingProjectVersions", () -> {
// TODO Pass this.scanner.version()
final List<VersionToScan> versionToScans = this.dao.versionsRequiringScans();
LocalDateTime start = LocalDateTime.now();
LOGGER.info("Rescanning {} versions", versionToScans.size());
for (final VersionToScan version : versionToScans) {
this.scan(version, false); // TODO partial parameter
}
// TODO partial parameter
versionToScans.forEach(version -> this.scan(version, false));
LOGGER.info("Rescanned. Took {}.", Duration.between(start, LocalDateTime.now()));
});
}

3
chart/values.yaml
View File

@ -39,6 +39,9 @@ ingress:
- path: /sitemap.xml
pathType: ImplementationSpecific
target: backend
- path: /total-sitemap.xml
pathType: ImplementationSpecific
target: backend
- path: /global-sitemap.xml
pathType: ImplementationSpecific
target: backend