mirror/Hangar
2
0
Fork 0
mirror of https://github.com/HangarMC/Hangar.git synced 2024-11-27 06:01:08 +08:00
Code Issues Packages Projects Releases Wiki Activity

Implement automated jarscanning on version upload

Browse Source
This commit is contained in:
Nassim Jahnke 2023-03-21 13:34:11 +01:00
parent 01b53d6905
commit 77f5fdae43
No known key found for this signature in database
GPG Key ID: 6BE3B555EBC5982B
10 changed files with 130 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
backend/src/main/java/io/papermc/hangar/db/dao/internal/table/versions/ProjectVersionReviewsDAO.java
View File

@ -18,7 +18,7 @@ public interface ProjectVersionReviewsDAO {
@Timestamped
@GetGeneratedKeys
@SqlUpdate("INSERT INTO project_version_reviews (created_at, version_id, user_id) VALUES (:now, :versionId, :userId)")
@SqlUpdate("INSERT INTO project_version_reviews (created_at, version_id, user_id, ended_at) VALUES (:now, :versionId, :userId, :endedAt)")
ProjectVersionReviewTable insert(@BindBean ProjectVersionReviewTable projectVersionReviewTable);
@Timestamped

4
backend/src/main/java/io/papermc/hangar/service/internal/users/UserService.java
View File

@ -47,6 +47,10 @@ public class UserService extends HangarComponent {
return this.getUserTable(userName, this.userDAO::getUserTable);
}
public @Nullable UserTable getUserTable(final UUID uuid) {
return this.getUserTable(uuid, this.userDAO::getUserTable);
}
public @Nullable UserTable getUserTable(final @Nullable Long userId) {
return this.getUserTable(userId, this.userDAO::getUserTable);
}

43
backend/src/main/java/io/papermc/hangar/service/internal/versions/JarScanningService.java
View File

@ -21,47 +21,79 @@ import io.papermc.hangar.service.internal.uploads.ProjectFiles;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.UUID;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import org.springframework.core.io.Resource;
import org.springframework.http.HttpStatus;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import software.amazon.awssdk.utils.ThreadFactoryBuilder;
@Service
public class JarScanningService {
public static final UUID JAR_SCANNER_USER = new UUID(952332837L, -376012533328L);
private static final ExecutorService EXECUTOR_SERVICE = Executors.newCachedThreadPool(new ThreadFactoryBuilder().threadNamePrefix("Scanner").build());
private final HangarJarScanner scanner = new HangarJarScanner();
private final JarScanResultDAO dao;
private final ProjectVersionsDAO projectVersionsDAO;
private final ProjectVersionDownloadsDAO downloadsDAO;
private final ProjectsDAO projectsDAO;
private final ProjectFiles projectFiles;
private final FileService fileService;
private final ReviewService reviewService;
public JarScanningService(final JarScanResultDAO dao, final ProjectVersionsDAO projectVersionsDAO, final ProjectVersionDownloadsDAO downloadsDAO, final ProjectsDAO projectsDAO, final ProjectFiles projectFiles, final FileService fileService) {
public JarScanningService(final JarScanResultDAO dao, final ProjectVersionsDAO projectVersionsDAO, final ProjectVersionDownloadsDAO downloadsDAO, final ProjectsDAO projectsDAO, final ProjectFiles projectFiles, final FileService fileService, final ReviewService reviewService) {
this.dao = dao;
this.projectVersionsDAO = projectVersionsDAO;
this.downloadsDAO = downloadsDAO;
this.projectsDAO = projectsDAO;
this.projectFiles = projectFiles;
this.fileService = fileService;
this.reviewService = reviewService;
}
public void scan(final long versionId, final Platform platform) {
public void scanAsync(final long versionId, final Collection<Platform> platforms) {
EXECUTOR_SERVICE.execute(() -> {
Severity highestSeverity = Severity.UNKNOWN;
for (final Platform platform : platforms) {
final Severity severity = this.scan(versionId, platform);
if (highestSeverity.compareTo(severity) < 0) {
highestSeverity = severity;
}
}
this.applyReview(highestSeverity, versionId);
});
}
@Transactional
void applyReview(final Severity severity, final long versionId) {
if (Severity.HIGH.compareTo(severity) < 0) {
this.reviewService.autoReview(versionId);
} else if (severity == Severity.HIGHEST) {
// TODO: state for requires changes or requires manual review
}
}
public Severity scan(final long versionId, final Platform platform) {
final Resource resource = this.getFile(versionId, platform);
final List<ScanResult> scanResults;
try (final InputStream inputStream = resource.getInputStream()) {
scanResults = this.scanner.scanJar(inputStream, resource.getFilename());
} catch (final IOException e) {
return;
return Severity.UNKNOWN;
}
// find the highest severity
Severity highestSeverity = Severity.UNKNOWN;
for (final ScanResult scanResult : scanResults) {
for (final Check.CheckResult result : scanResult.results()) {
if (highestSeverity.compareTo(result.severity()) > 0) {
if (highestSeverity.compareTo(result.severity()) < 0) {
highestSeverity = result.severity();
}
}
@ -75,6 +107,7 @@ public class JarScanningService {
}
this.dao.save(new JarScanResultTable(versionId, platform, highestSeverity.name(), new JSONB(formattedResults)));
return highestSeverity;
}
private Resource getFile(final long versionId, final Platform platform) {

47
backend/src/main/java/io/papermc/hangar/service/internal/versions/ReviewService.java
View File

@ -9,6 +9,7 @@ import io.papermc.hangar.exceptions.HangarApiException;
import io.papermc.hangar.model.common.ReviewAction;
import io.papermc.hangar.model.common.projects.ReviewState;
import io.papermc.hangar.model.common.projects.Visibility;
import io.papermc.hangar.model.db.UserTable;
import io.papermc.hangar.model.db.versions.ProjectVersionTable;
import io.papermc.hangar.model.db.versions.reviews.ProjectVersionReviewMessageTable;
import io.papermc.hangar.model.db.versions.reviews.ProjectVersionReviewTable;
@ -18,10 +19,12 @@ import io.papermc.hangar.model.internal.logs.contexts.VersionContext;
import io.papermc.hangar.model.internal.versions.HangarReview;
import io.papermc.hangar.model.internal.versions.HangarReviewQueueEntry;
import io.papermc.hangar.service.internal.users.NotificationService;
import io.papermc.hangar.service.internal.users.UserService;
import io.papermc.hangar.service.internal.visibility.ProjectVersionVisibilityService;
import jakarta.validation.constraints.NotNull;
import java.time.OffsetDateTime;
import java.util.List;
import java.util.Locale;
import java.util.Objects;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpStatus;
@ -37,14 +40,35 @@ public class ReviewService extends HangarComponent {
private final ProjectVersionsDAO projectVersionsDAO;
private final ProjectVersionVisibilityService projectVersionVisibilityService;
private final NotificationService notificationService;
private final UserService userService;
@Autowired
public ReviewService(final ProjectVersionReviewsDAO projectVersionReviewsDAO, final HangarReviewsDAO hangarReviewsDAO, final ProjectVersionsDAO projectVersionsDAO, final ProjectVersionVisibilityService projectVersionVisibilityService, final NotificationService notificationService) {
public ReviewService(final ProjectVersionReviewsDAO projectVersionReviewsDAO, final HangarReviewsDAO hangarReviewsDAO, final ProjectVersionsDAO projectVersionsDAO, final ProjectVersionVisibilityService projectVersionVisibilityService, final NotificationService notificationService, final UserService userService) {
this.projectVersionReviewsDAO = projectVersionReviewsDAO;
this.hangarReviewsDAO = hangarReviewsDAO;
this.projectVersionsDAO = projectVersionsDAO;
this.projectVersionVisibilityService = projectVersionVisibilityService;
this.notificationService = notificationService;
this.userService = userService;
this.initJarScannerUser();
}
private void initJarScannerUser() {
if (this.userService.getUserTable(JarScanningService.JAR_SCANNER_USER) != null) {
return;
}
final UserTable userTable = new UserTable(
-1,
JarScanningService.JAR_SCANNER_USER,
"JarScanner",
"automated@test.test",
List.of(),
false,
Locale.ENGLISH.toLanguageTag(),
"white"
);
this.userService.insertUser(userTable);
}
public List<HangarReview> getHangarReviews(final long versionId) {
@ -62,6 +86,27 @@ public class ReviewService extends HangarComponent {
this.changeVersionReviewState(versionId, ReviewState.UNDER_REVIEW, false);
}
@Transactional
public void autoReview(final long versionId) {
final UserTable jarScannerUser = this.userService.getUserTable(JarScanningService.JAR_SCANNER_USER);
ProjectVersionReviewTable projectVersionReviewTable = new ProjectVersionReviewTable(versionId, jarScannerUser.getUserId());
projectVersionReviewTable.setEndedAt(OffsetDateTime.now());
projectVersionReviewTable = this.projectVersionReviewsDAO.insert(projectVersionReviewTable);
this.projectVersionReviewsDAO.insertMessage(new ProjectVersionReviewMessageTable(
projectVersionReviewTable.getId(),
"[AUTO] Automated review",
new JSONB("{}"),
ReviewAction.PARTIALLY_APPROVE
));
final ProjectVersionTable projectVersionTable = this.projectVersionsDAO.getProjectVersionTable(versionId);
final ReviewState oldState = projectVersionTable.getReviewState();
if (oldState != ReviewState.REVIEWED) {
projectVersionTable.setReviewState(ReviewState.PARTIALLY_REVIEWED);
this.projectVersionsDAO.update(projectVersionTable);
}
}
@Transactional
public void addReviewMessage(final long versionId, final ReviewMessage msg) {
final ProjectVersionReviewTable latestUnfinishedReview = this.getLatestUnfinishedReviewAndValidate(versionId);

12
backend/src/main/java/io/papermc/hangar/service/internal/versions/VersionFactory.java
View File

@ -87,9 +87,10 @@ public class VersionFactory extends HangarComponent {
private final ProjectVersionDownloadsDAO downloadsDAO;
private final VersionsApiDAO versionsApiDAO;
private final FileService fileService;
private final JarScanningService jarScanningService;
@Autowired
public VersionFactory(final ProjectVersionPlatformDependenciesDAO projectVersionPlatformDependencyDAO, final ProjectVersionDependenciesDAO projectVersionDependencyDAO, final ProjectVersionsDAO projectVersionDAO, final ProjectFiles projectFiles, final PluginDataService pluginDataService, final ChannelService channelService, final ProjectVisibilityService projectVisibilityService, final ProjectService projectService, final NotificationService notificationService, final PlatformService platformService, final UsersApiService usersApiService, final JobService jobService, final ValidationService validationService, final ProjectVersionDownloadsDAO downloadsDAO, final VersionsApiDAO versionsApiDAO, final FileService fileService) {
public VersionFactory(final ProjectVersionPlatformDependenciesDAO projectVersionPlatformDependencyDAO, final ProjectVersionDependenciesDAO projectVersionDependencyDAO, final ProjectVersionsDAO projectVersionDAO, final ProjectFiles projectFiles, final PluginDataService pluginDataService, final ChannelService channelService, final ProjectVisibilityService projectVisibilityService, final ProjectService projectService, final NotificationService notificationService, final PlatformService platformService, final UsersApiService usersApiService, final JobService jobService, final ValidationService validationService, final ProjectVersionDownloadsDAO downloadsDAO, final VersionsApiDAO versionsApiDAO, final FileService fileService, final JarScanningService jarScanningService) {
this.projectVersionPlatformDependenciesDAO = projectVersionPlatformDependencyDAO;
this.projectVersionDependenciesDAO = projectVersionDependencyDAO;
this.projectVersionsDAO = projectVersionDAO;
@ -106,6 +107,7 @@ public class VersionFactory extends HangarComponent {
this.downloadsDAO = downloadsDAO;
this.versionsApiDAO = versionsApiDAO;
this.fileService = fileService;
this.jarScanningService = jarScanningService;
}
@Transactional
@ -307,6 +309,14 @@ public class VersionFactory extends HangarComponent {
// cache purging
this.projectService.refreshHomeProjects();
this.usersApiService.clearAuthorsCache();
final List<Platform> platformsToScan = new ArrayList<>();
for (final PendingVersionFile file : pendingVersion.getFiles()) {
if (file.fileInfo() != null) {
platformsToScan.add(file.platforms().get(0));
}
}
this.jarScanningService.scanAsync(projectVersionTable.getVersionId(), platformsToScan);
} catch (final HangarApiException e) {
throw e;
} catch (final Exception e) {

14
backend/src/main/resources/db/migration/V1.4.1__yeet_previous_scans.sql Normal file
View File

@ -0,0 +1,14 @@
DROP TABLE jar_scan_result;
CREATE TABLE jar_scan_result
(
id bigserial NOT NULL
CONSTRAINT jar_scan_result_pkey
PRIMARY KEY,
version_id integer NOT NULL
CONSTRAINT jar_scan_result_version_id_fk
REFERENCES project_versions ON DELETE CASCADE,
platform bigint NOT NULL,
data jsonb NOT NULL,
highest_severity text NOT NULL,
created_at timestamp WITH TIME ZONE NOT NULL
);

4
frontend/src/components/projects/DownloadButton.vue
View File

@ -10,6 +10,7 @@ import { useBackendData } from "~/store/backendData";
import DropdownItem from "~/lib/components/design/DropdownItem.vue";
import PlatformLogo from "~/components/logos/platforms/PlatformLogo.vue";
import { useInternalApi } from "~/composables/useApi";
import { formatSize } from "~/lib/composables/useFile";
const i18n = useI18n();
@ -28,6 +29,7 @@ const props = withDefaults(
platform?: Platform;
version?: DownloadableVersion;
pinnedVersion?: PinnedVersion;
showFileSize?: boolean;
}>(),
{
small: false,
@ -36,6 +38,7 @@ const props = withDefaults(
platform: undefined,
version: undefined,
pinnedVersion: undefined,
showFileSize: false,
}
);
@ -141,6 +144,7 @@ function trackDownload(platform: Platform, version: DownloadableVersion & { id?:
<PlatformLogo :platform="p" :size="24" class="mr-1 flex-shrink-0" />
{{ useBackendData.platforms.get(p)?.name }}
<span v-if="showVersions && version.platformDependencies" class="ml-1">({{ version.platformDependenciesFormatted[p] }})</span>
<span v-if="v.fileInfo?.sizeBytes" class="ml-1"> ({{ formatSize(v.fileInfo.sizeBytes) }}) </span>
</DropdownItem>
</DropdownButton>

8
frontend/src/pages/[user]/[project]/versions/[version]/index.vue
View File

@ -158,13 +158,9 @@ async function restoreVersion() {
<h3>
<span class="inline-flex lt-md:flex-wrap">
{{ i18n.t("version.page.subheader", [projectVersion.author, lastUpdated(new Date(projectVersion.createdAt))]) }}
<!--<span v-if="projectVersion.downloads[platform?.enumName]?.fileInfo?.sizeBytes" class="inline-flex items-center sm:ml-3">
<IconMdiFile class="mr-1" />
{{ filesize(projectVersion.downloads[platform.enumName].fileInfo.sizeBytes) }}
</span>-->
</span>
</h3>
<em v-if="hasPerms(NamedPermission.REVIEWER) && projectVersion.approvedBy" class="text-lg ml-1">
<em v-if="hasPerms(NamedPermission.REVIEWER) && projectVersion.approvedBy">
{{ i18n.t("version.page.adminMsg", [projectVersion.approvedBy, i18n.d(projectVersion.createdAt, "date")]) }}
</em>
</div>
@ -173,7 +169,7 @@ async function restoreVersion() {
<Tooltip v-if="confirmationWarningKey" :content="i18n.t(confirmationWarningKey)">
<IconMdiAlertCircleOutline class="text-2xl" />
</Tooltip>
<DownloadButton :version="projectVersion" :project="project" :show-single-platform="false" :show-versions="false" />
<DownloadButton :version="projectVersion" :project="project" :show-single-platform="false" :show-versions="false" show-file-size />
</div>
</div>

9
frontend/src/pages/[user]/[project]/versions/[version]/scan.vue
View File

@ -20,6 +20,10 @@ const route = useRoute();
const results = ref<JarScanResult[]>([]);
for (const platform of props.versionPlatforms) {
if (!props.version.downloads[platform]?.fileInfo) {
continue;
}
const result = await useJarScan(props.version.id, platform);
if (result.value) {
results.value.push(result.value);
@ -39,7 +43,10 @@ useHead(useSeo("Scan | " + props.project.name, props.project.description, route,
<template>
<div v-if="version" class="mt-4">
<div v-for="result in results" :key="result.id" class="mb-4">
<h3 class="text-xl">Results for {{ version.name }} {{ result.platform }} (last scanned at <PrettyTime :time="result.createdAt" short-relative />)</h3>
<h3 class="text-2xl inline-flex items-center space-x-1">
<IconMdiInformation class="mr-1" />
Results for {{ version.name }} {{ result.platform }} (last scanned: <PrettyTime :time="result.createdAt" short-relative />)
</h3>
<div v-for="(file, idx) in result.data" :key="idx">
<div v-for="(line, idx2) in file" :key="idx2">
{{ line }}

1
frontend/src/types/generated/icons.d.ts vendored
View File

@ -46,6 +46,7 @@ declare module "@vue/runtime-core" {
IconMdiEye: typeof import("~icons/mdi/eye")["default"];
IconMdiEyeOff: typeof import("~icons/mdi/eye-off")["default"];
IconMdiFeather: typeof import("~icons/mdi/feather")["default"];
IconMdiFile: typeof import("~icons/mdi/file")["default"];
IconMdiFlag: typeof import("~icons/mdi/flag")["default"];
IconMdiFolderPlusOutline: typeof import("~icons/mdi/folder-plus-outline")["default"];
IconMdiFormatListNumbered: typeof import("~icons/mdi/format-list-numbered")["default"];