Port: Improve performance of findings retrieval

src/main/java/org/dependencytrack/model/Finding.java

@@ -88,7 +88,7 @@ public class Finding implements Serializable {
                 ON "COMPONENT"."ID" = "COMPONENTS_VULNERABILITIES"."COMPONENT_ID"
              INNER JOIN "VULNERABILITY"
                 ON "COMPONENTS_VULNERABILITIES"."VULNERABILITY_ID" = "VULNERABILITY"."ID"
-             LEFT JOIN "EPSS"
+              LEFT JOIN "EPSS"
                 ON "VULNERABILITY"."VULNID" = "EPSS"."CVE"
              INNER JOIN "FINDINGATTRIBUTION"
                 ON "COMPONENT"."ID" = "FINDINGATTRIBUTION"."COMPONENT_ID"
@@ -97,7 +97,8 @@ public class Finding implements Serializable {
                 ON "COMPONENT"."ID" = "ANALYSIS"."COMPONENT_ID"
                AND "VULNERABILITY"."ID" = "ANALYSIS"."VULNERABILITY_ID"
                AND "COMPONENT"."PROJECT_ID" = "ANALYSIS"."PROJECT_ID"
-             WHERE "COMPONENT"."PROJECT_ID" = ?
+             WHERE "COMPONENT"."PROJECT_ID" = :projectId
+               AND (:includeSuppressed OR "ANALYSIS"."SUPPRESSED" IS NULL OR NOT "ANALYSIS"."SUPPRESSED")
             """;
 
     // language=SQL
@@ -179,8 +180,8 @@ public Finding(UUID project, Object... o) {
         optValue(vulnerability, "vulnId", o[8]);
         optValue(vulnerability, "title", o[9]);
         optValue(vulnerability, "subtitle", o[10]);
-        //optValue(vulnerability, "description", o[11]); // CLOB - handle this in QueryManager
-        //optValue(vulnerability, "recommendation", o[12]); // CLOB - handle this in QueryManager
+        optValue(vulnerability, "description", o[11]);
+        optValue(vulnerability, "recommendation", o[12]);
         final Severity severity = VulnerabilityUtil.getSeverity(o[13], (BigDecimal) o[14], (BigDecimal) o[15], (BigDecimal) o[16], (BigDecimal) o[17], (BigDecimal) o[18]);
         optValue(vulnerability, "cvssV2BaseScore", o[14]);
         optValue(vulnerability, "cvssV3BaseScore", o[15]);
@@ -295,4 +296,5 @@ public void addVulnerabilityAliases(List<VulnerabilityAlias> aliases) {
         }
         vulnerability.put("aliases",uniqueAliases);
     }
+
 }

src/main/java/org/dependencytrack/model/VulnIdAndSource.java

@@ -0,0 +1,32 @@
+/*
+ * This file is part of Dependency-Track.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright (c) OWASP Foundation. All Rights Reserved.
+ */
+package org.dependencytrack.model;
+
+/**
+ * @param vulnId {@link Vulnerability#getVulnId()}
+ * @param source {@link Vulnerability#getSource()}
+ * @since 4.12.0
+ */
+public record VulnIdAndSource(String vulnId, Vulnerability.Source source) {
+
+    public VulnIdAndSource(String vulnId, String source) {
+        this(vulnId, Vulnerability.Source.valueOf(source));
+    }
+
+}

src/main/java/org/dependencytrack/persistence/FindingsQueryManager.java

@@ -28,16 +28,22 @@
 import org.dependencytrack.model.Component;
 import org.dependencytrack.model.Finding;
 import org.dependencytrack.model.Project;
-import org.dependencytrack.model.RepositoryMetaComponent;
 import org.dependencytrack.model.RepositoryType;
+import org.dependencytrack.model.VulnIdAndSource;
 import org.dependencytrack.model.Vulnerability;
 import org.dependencytrack.model.VulnerabilityAlias;
+import org.dependencytrack.persistence.RepositoryQueryManager.RepositoryMetaComponentSearch;
+import org.dependencytrack.util.PurlUtil;
 
 import javax.jdo.PersistenceManager;
 import javax.jdo.Query;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.Date;
 import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.stream.Collectors;
 
 public class FindingsQueryManager extends QueryManager implements IQueryManager {
 
@@ -338,33 +344,68 @@ public List<Finding> getFindings(Project project) {
     @SuppressWarnings("unchecked")
     public List<Finding> getFindings(Project project, boolean includeSuppressed) {
         final Query<Object[]> query = pm.newQuery(Query.SQL, Finding.QUERY);
-        query.setParameters(project.getId());
-        final List<Object[]> list = query.executeList();
-        final List<Finding> findings = new ArrayList<>();
-        for (final Object[] o : list) {
-            final Finding finding = new Finding(project.getUuid(), o);
-            final Component component = getObjectByUuid(Component.class, (String) finding.getComponent().get("uuid"));
-            final Vulnerability vulnerability = getObjectByUuid(Vulnerability.class, (String) finding.getVulnerability().get("uuid"));
-            final Analysis analysis = getAnalysis(component, vulnerability);
-            final List<VulnerabilityAlias> aliases = detach(getVulnerabilityAliases(vulnerability));
-            finding.addVulnerabilityAliases(aliases);
-            if (includeSuppressed || analysis == null || !analysis.isSuppressed()) { // do not add globally suppressed findings
-                // These are CLOB fields. Handle these here so that database-specific deserialization doesn't need to be performed (in Finding)
-                finding.getVulnerability().put("description", vulnerability.getDescription());
-                finding.getVulnerability().put("recommendation", vulnerability.getRecommendation());
-                final PackageURL purl = component.getPurl();
-                if (purl != null) {
-                    final RepositoryType type = RepositoryType.resolve(purl);
-                    if (RepositoryType.UNSUPPORTED != type) {
-                        final RepositoryMetaComponent repoMetaComponent = getRepositoryMetaComponent(type, purl.getNamespace(), purl.getName());
-                        if (repoMetaComponent != null) {
-                            finding.getComponent().put("latestVersion", repoMetaComponent.getLatestVersion());
-                        }
-                    }
-                }
-                findings.add(finding);
+        query.setNamedParameters(Map.ofEntries(
+                Map.entry("projectId", project.getId()),
+                Map.entry("includeSuppressed", includeSuppressed)
+        ));
+        final List<Object[]> queryResultRows;
+        try {
+            queryResultRows = new ArrayList<>(query.executeList());
+        } finally {
+            query.closeAll();
+        }
+
+        final List<Finding> findings = queryResultRows.stream()
+                .map(row -> new Finding(project.getUuid(), row))
+                .toList();
+
+        final Map<VulnIdAndSource, List<Finding>> findingsByVulnIdAndSource = findings.stream()
+                .collect(Collectors.groupingBy(
+                        finding -> new VulnIdAndSource(
+                                (String) finding.getVulnerability().get("vulnId"),
+                                (String) finding.getVulnerability().get("source")
+                        )
+                ));
+        final Map<VulnIdAndSource, List<VulnerabilityAlias>> aliasesByVulnIdAndSource =
+                getVulnerabilityAliases(findingsByVulnIdAndSource.keySet());
+        for (final VulnIdAndSource vulnIdAndSource : findingsByVulnIdAndSource.keySet()) {
+            final List<Finding> affectedFindings = findingsByVulnIdAndSource.get(vulnIdAndSource);
+            final List<VulnerabilityAlias> aliases = aliasesByVulnIdAndSource.getOrDefault(vulnIdAndSource, Collections.emptyList());
+
+            for (final Finding finding : affectedFindings) {
+                finding.addVulnerabilityAliases(aliases);
             }
         }
+
+        final Map<RepositoryMetaComponentSearch, List<Finding>> findingsByMetaComponentSearch = findings.stream()
+                .filter(finding -> finding.getComponent().get("purl") != null)
+                .map(finding -> {
+                    final PackageURL purl = PurlUtil.silentPurl((String) finding.getComponent().get("purl"));
+                    if (purl == null) {
+                        return null;
+                    }
+
+                    final var repositoryType = RepositoryType.resolve(purl);
+                    if (repositoryType == RepositoryType.UNSUPPORTED) {
+                        return null;
+                    }
+
+                    final var search = new RepositoryMetaComponentSearch(repositoryType, purl.getNamespace(), purl.getName());
+                    return Map.entry(search, finding);
+                })
+                .filter(Objects::nonNull)
+                .collect(Collectors.groupingBy(
+                        Map.Entry::getKey,
+                        Collectors.mapping(Map.Entry::getValue, Collectors.toList())
+                ));
+        getRepositoryMetaComponents(List.copyOf(findingsByMetaComponentSearch.keySet()))
+                .forEach(metaComponent -> {
+                    final var search = new RepositoryMetaComponentSearch(metaComponent.getRepositoryType(), metaComponent.getNamespace(), metaComponent.getName());
+                    final List<Finding> affectedFindings = findingsByMetaComponentSearch.get(search);
+                    for (final Finding finding : affectedFindings) {
+                        finding.getComponent().put("latestVersion", metaComponent.getLatestVersion());
+                    }
+                });
         return findings;
     }
 }

src/main/java/org/dependencytrack/persistence/QueryManager.java

@@ -80,6 +80,7 @@
 import org.dependencytrack.model.ViolationAnalysis;
 import org.dependencytrack.model.ViolationAnalysisComment;
 import org.dependencytrack.model.ViolationAnalysisState;
+import org.dependencytrack.model.VulnIdAndSource;
 import org.dependencytrack.model.Vulnerability;
 import org.dependencytrack.model.VulnerabilityAlias;
 import org.dependencytrack.model.VulnerabilityMetrics;
@@ -1153,6 +1154,10 @@ public List<VulnerabilityAlias> getVulnerabilityAliases(Vulnerability vulnerabil
         return getVulnerabilityQueryManager().getVulnerabilityAliases(vulnerability);
     }
 
+    public Map<VulnIdAndSource, List<VulnerabilityAlias>> getVulnerabilityAliases(final Collection<VulnIdAndSource> vulnIdAndSources) {
+        return getVulnerabilityQueryManager().getVulnerabilityAliases(vulnIdAndSources);
+    }
+
     List<Analysis> getAnalyses(Project project) {
         return getFindingsQueryManager().getAnalyses(project);
     }
@@ -1612,6 +1617,10 @@ public List<RepositoryMetaComponent> getRepositoryMetaComponentsBatch(final List
         return results;
     }
 
+    public List<RepositoryMetaComponent> getRepositoryMetaComponents(final List<RepositoryQueryManager.RepositoryMetaComponentSearch> list) {
+        return getRepositoryQueryManager().getRepositoryMetaComponents(list);
+    }
+
     /**
      * Create a new {@link VulnerabilityScan} record.
      * <p>

src/main/java/org/dependencytrack/persistence/VulnerabilityQueryManager.java

@@ -30,6 +30,7 @@
 import org.dependencytrack.model.FindingAttribution;
 import org.dependencytrack.model.Project;
 import org.dependencytrack.model.Tag;
+import org.dependencytrack.model.VulnIdAndSource;
 import org.dependencytrack.model.Vulnerability;
 import org.dependencytrack.model.VulnerabilityAlias;
 import org.dependencytrack.model.VulnerableSoftware;
@@ -776,6 +777,94 @@
         return (List<VulnerabilityAlias>)query.execute(vulnerability.getVulnId());
     }
 
+    /**
+     * Bulk-load {@link VulnerabilityAlias}es for one or more {@link VulnIdAndSource}s.
+     *
+     * @param vulnIdAndSources The Vulnerability ID - Source pairs to load {@link VulnerabilityAlias}es for
+     * @return {@link VulnerabilityAlias}es, grouped by {@link VulnIdAndSource}
+     * @since 4.12.0
+     */
+    @Override
+    public Map<VulnIdAndSource, List<VulnerabilityAlias>> getVulnerabilityAliases(final Collection<VulnIdAndSource> vulnIdAndSources) {
+        if (vulnIdAndSources == null || vulnIdAndSources.isEmpty()) {
+            return Collections.emptyMap();
+        }
+
+        var subQueryIndex = 0;
+        final var subQueries = new ArrayList<String>(vulnIdAndSources.size());
+        final var params = new HashMap<String, Object>(vulnIdAndSources.size());
+        for (final VulnIdAndSource vulnIdAndSource : vulnIdAndSources) {
+            subQueryIndex++;
+            final String vulnIdParamName = "vulnId" + subQueryIndex;
+
+            final String filter = switch (vulnIdAndSource.source()) {
+                case GITHUB -> "\"GHSA_ID\" = :" + vulnIdParamName;
+                case INTERNAL -> "\"INTERNAL_ID\" = :" + vulnIdParamName;
+                case NVD -> "\"CVE_ID\" = :" + vulnIdParamName;
+                case OSSINDEX -> "\"SONATYPE_ID\" = :" + vulnIdParamName;
+                case OSV -> "\"OSV_ID\" = :" + vulnIdParamName;
+                case SNYK -> "\"SNYK_ID\" = :" + vulnIdParamName;
+                case VULNDB -> "\"VULNDB_ID\" = :" + vulnIdParamName;
+                default -> null;
+            };
+            if (filter == null) {
+                continue;
+            }
+
+            // We'll need to correlate query results with VulnIdAndSource objects
+            // later, so prepend an identifier for them to the result set.
+            // NB: DataNucleus doesn't support usage of query parameters
+            // in the SELECT statement. Use hashCode of vulnIdAndSource instead
+            // of string literals (which could be susceptible to SQL injection).
+            subQueries.add("""
+                    SELECT %d
+                         , "GHSA_ID"
+                         , "INTERNAL_ID"
+                         , "CVE_ID"
+                         , "SONATYPE_ID"
+                         , "OSV_ID"
+                         , "SNYK_ID"
+                         , "VULNDB_ID"
+                      FROM "VULNERABILITYALIAS"
+                     WHERE %s
+                    """.formatted(vulnIdAndSource.hashCode(), filter));
+            params.put(vulnIdParamName, vulnIdAndSource.vulnId());
+        }
+
+        final Query<Object[]> query = pm.newQuery(Query.SQL, String.join(" UNION ALL ", subQueries));
+        query.setNamedParameters(params);
+        final List<Object[]> queryResultRows;
+        try {
+            queryResultRows = new ArrayList<>(query.executeList());
+        } finally {
+            query.closeAll();
+        }
+        if (queryResultRows.isEmpty()) {
+            return Collections.emptyMap();
+        }
+
+        final Map<Integer, VulnIdAndSource> vulnIdAndSourceByHashCode = vulnIdAndSources.stream()
+                .collect(Collectors.toMap(Record::hashCode, Function.identity()));
+
+        return queryResultRows.stream()
+                .map(row -> {
+                    final var vulnIdAndSource = vulnIdAndSourceByHashCode.get((Integer) row[0]);
+                    final var alias = new VulnerabilityAlias();
+                    alias.setGhsaId((String) row[1]);
+                    alias.setInternalId((String) row[2]);
+                    alias.setCveId((String) row[3]);
+                    alias.setSonatypeId((String) row[4]);
+                    alias.setOsvId((String) row[5]);
+                    alias.setSnykId((String) row[6]);
+                    alias.setVulnDbId((String) row[7]);
+                    return Map.entry(vulnIdAndSource, alias);
+                })
+                .collect(Collectors.groupingBy(
+                        Map.Entry::getKey,
+                        Collectors.mapping(Map.Entry::getValue, Collectors.toList())
+                ));
+    }
+
     /**
      * Reconcile {@link VulnerableSoftware} for a given {@link Vulnerability}.
      * <p>

src/main/java/org/dependencytrack/util/PurlUtil.java

@@ -52,4 +52,23 @@ public static PackageURL silentPurlCoordinatesOnly(final PackageURL original) {
         }
     }
 
+    /**
+     * Attempt to parse a given package URL.
+     *
+     * @param purl The package URL to parse
+     * @return The parsed {@link PackageURL}, or {@code null} when parsing failed
+     * @since 4.12.0
+     */
+    public static PackageURL silentPurl(final String purl) {
+        if (purl == null) {
+            return null;
+        }
+
+        try {
+            return new PackageURL(purl);
+        } catch (MalformedPackageURLException ignored) {
+            return null;
+        }
+    }
+
 }

src/test/java/org/dependencytrack/util/PurlUtilTest.java

@@ -0,0 +1,52 @@
+/*
+ * This file is part of Dependency-Track.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright (c) OWASP Foundation. All Rights Reserved.
+ */
+package org.dependencytrack.util;
+
+import com.github.packageurl.PackageURL;
+import org.junit.Test;
+
+import java.util.Map;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class PurlUtilTest {
+
+    @Test
+    public void testSilentPurlWithNull() {
+        assertThat(PurlUtil.silentPurl(null)).isNull();
+    }
+
+    @Test
+    public void testSilentPurlWithInvalidPurl() {
+        assertThat(PurlUtil.silentPurl("foo:bar:baz")).isNull();
+    }
+
+    @Test
+    public void testSilentPurlWithValidPurl() {
+        final PackageURL purl = PurlUtil.silentPurl("pkg:maven/foo/[email protected]?qux=quux#baz");
+        assertThat(purl).isNotNull();
+        assertThat(purl.getType()).isEqualTo("maven");
+        assertThat(purl.getNamespace()).isEqualTo("foo");
+        assertThat(purl.getName()).isEqualTo("bar");
+        assertThat(purl.getVersion()).isEqualTo("1.2.3");
+        assertThat(purl.getSubpath()).isEqualTo("baz");
+        assertThat(purl.getQualifiers()).containsOnly(Map.entry("qux", "quux"));
+    }
+
+}