Skip to content

Commit

Permalink
Merge pull request #636 from DependencyTrack/epss-mirroring
Browse files Browse the repository at this point in the history
Epss mirroring
  • Loading branch information
nscuro authored Apr 17, 2024
2 parents 0e03953 + 73f74d2 commit fee43d8
Show file tree
Hide file tree
Showing 44 changed files with 809 additions and 368 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/_meta-build.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -156,4 +156,4 @@ jobs:
if: ${{ inputs.publish-container }}
uses: github/codeql-action/upload-sarif@v3
with:
sarif_file: 'trivy-results.sarif'
sarif_file: 'trivy-results.sarif'
2 changes: 1 addition & 1 deletion .github/workflows/ci-build.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -48,4 +48,4 @@ jobs:
security-events: write # Required to upload trivy's SARIF output
secrets:
registry-0-usr: ${{ github.repository_owner }}
registry-0-psw: ${{ secrets.GITHUB_TOKEN }}
registry-0-psw: ${{ secrets.GITHUB_TOKEN }}
2 changes: 1 addition & 1 deletion .github/workflows/ci-publish.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -94,4 +94,4 @@ jobs:
--clobber \
target/dependency-track-apiserver.jar \
target/checksums.txt \
target/bom.json
target/bom.json
2 changes: 1 addition & 1 deletion .github/workflows/ci-test.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -91,4 +91,4 @@ jobs:
path: |-
pr-commit.txt
pr-number.txt
target/jacoco-ut/jacoco.xml
target/jacoco-ut/jacoco.xml
1 change: 1 addition & 0 deletions src/main/java/org/dependencytrack/common/ConfigKey.java
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ public enum ConfigKey implements Config.Key {
CRON_EXPRESSION_FOR_PORTFOLIO_METRICS_TASK("task.cron.metrics.portfolio", "10 * * * *"),
CRON_EXPRESSION_FOR_VULNERABILITY_METRICS_TASK("task.cron.metrics.vulnerability", "40 * * * *"),
CRON_EXPRESSION_FOR_COMPONENT_IDENTIFICATION_TASK("task.cron.componentIdentification", "25 */6 * * *"),
CRON_EXPRESSION_FOR_EPSS_MIRRORING_TASK("task.cron.mirror.epss", "0 1 * * *"),
CRON_EXPRESSION_FOR_GITHUB_MIRRORING_TASK("task.cron.mirror.github", "0 2 * * *"),
CRON_EXPRESSION_FOR_OSV_MIRRORING_TASK("task.cron.mirror.osv", "0 3 * * *"),
CRON_EXPRESSION_FOR_NIST_MIRRORING_TASK("task.cron.mirror.nist", "0 4 * * *"),
Expand Down
13 changes: 2 additions & 11 deletions src/main/java/org/dependencytrack/event/EpssMirrorEvent.java
Original file line number Diff line number Diff line change
Expand Up @@ -18,22 +18,13 @@
*/
package org.dependencytrack.event;

import alpine.event.framework.SingletonCapableEvent;

import java.util.UUID;
import alpine.event.framework.Event;

/**
* Defines an event used to start a mirror of EPSS.
*
* @author Steve Springett
* @since 4.5.0
*/
public class EpssMirrorEvent extends SingletonCapableEvent {

private static final UUID CHAIN_IDENTIFIER = UUID.fromString("63aa687a-17f0-4e2d-abd3-e2016b3c4f0a");

public EpssMirrorEvent() {
setChainIdentifier(CHAIN_IDENTIFIER);
setSingleton(true);
}
public class EpssMirrorEvent implements Event {
}
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import com.google.protobuf.Message;
import org.dependencytrack.event.ComponentRepositoryMetaAnalysisEvent;
import org.dependencytrack.event.ComponentVulnerabilityAnalysisEvent;
import org.dependencytrack.event.EpssMirrorEvent;
import org.dependencytrack.event.GitHubAdvisoryMirrorEvent;
import org.dependencytrack.event.NistMirrorEvent;
import org.dependencytrack.event.OsvMirrorEvent;
Expand Down Expand Up @@ -69,6 +70,7 @@ private KafkaEventConverter() {
case GitHubAdvisoryMirrorEvent e -> convert(e);
case NistMirrorEvent e -> convert(e);
case OsvMirrorEvent e -> convert(e);
case EpssMirrorEvent e -> convert(e);
default -> throw new IllegalArgumentException("Unable to convert event " + event);
};
}
Expand Down Expand Up @@ -159,6 +161,10 @@ static KafkaEvent<String, String> convert(final OsvMirrorEvent event) {
return new KafkaEvent<>(KafkaTopics.VULNERABILITY_MIRROR_COMMAND, key, value);
}

static KafkaEvent<String, String> convert(final EpssMirrorEvent ignored) {
return new KafkaEvent<>(KafkaTopics.VULNERABILITY_MIRROR_COMMAND, "EPSS", null);
}

private static Topic<String, Notification> extractDestinationTopic(final Notification notification) {
return switch (notification.getGroup()) {
case GROUP_ANALYZER -> KafkaTopics.NOTIFICATION_ANALYZER;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -128,5 +128,4 @@ final var record = new ProducerRecord<>(event.topic().name(), keyBytes, valueByt

return record;
}

}
}
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import org.cyclonedx.proto.v1_4.Bom;
import org.dependencytrack.common.ConfigKey;
import org.dependencytrack.event.kafka.serialization.KafkaProtobufSerde;
import org.dependencytrack.proto.mirror.v1.EpssItem;
import org.dependencytrack.proto.notification.v1.Notification;
import org.dependencytrack.proto.repometaanalysis.v1.AnalysisCommand;
import org.dependencytrack.proto.repometaanalysis.v1.AnalysisResult;
Expand Down Expand Up @@ -54,6 +55,7 @@ public final class KafkaTopics {
public static final Topic<ScanKey, ScanResult> VULN_ANALYSIS_RESULT;

public static final Topic<String, Notification> NOTIFICATION_PROJECT_VULN_ANALYSIS_COMPLETE;
public static final Topic<String, EpssItem> NEW_EPSS;
private static final Serde<Notification> NOTIFICATION_SERDE = new KafkaProtobufSerde<>(Notification.parser());

static {
Expand All @@ -77,6 +79,7 @@ public final class KafkaTopics {
VULN_ANALYSIS_COMMAND = new Topic<>("dtrack.vuln-analysis.component", new KafkaProtobufSerde<>(ScanKey.parser()), new KafkaProtobufSerde<>(ScanCommand.parser()));
VULN_ANALYSIS_RESULT = new Topic<>("dtrack.vuln-analysis.result", new KafkaProtobufSerde<>(ScanKey.parser()), new KafkaProtobufSerde<>(ScanResult.parser()));
NOTIFICATION_PROJECT_VULN_ANALYSIS_COMPLETE = new Topic<>("dtrack.notification.project-vuln-analysis-complete", Serdes.String(), NOTIFICATION_SERDE);
NEW_EPSS = new Topic<>("dtrack.epss", Serdes.String(), new KafkaProtobufSerde<>(EpssItem.parser()));
}

public record Topic<K, V>(String name, Serde<K> keySerde, Serde<V> valueSerde) {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
/*
* This file is part of Dependency-Track.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
* Copyright (c) OWASP Foundation. All Rights Reserved.
*/
package org.dependencytrack.event.kafka.processor;

import alpine.common.logging.Logger;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.dependencytrack.event.kafka.processor.api.BatchProcessor;
import org.dependencytrack.event.kafka.processor.exception.ProcessingException;
import org.dependencytrack.model.Epss;
import org.dependencytrack.parser.dependencytrack.EpssModelConverter;
import org.dependencytrack.persistence.QueryManager;
import org.dependencytrack.proto.mirror.v1.EpssItem;

import java.util.List;


public class EpssMirrorProcessor implements BatchProcessor<String, EpssItem> {

public static final String PROCESSOR_NAME = "epss.mirror";
private static final Logger LOGGER = Logger.getLogger(EpssMirrorProcessor.class);

@Override
public void process(List<ConsumerRecord<String, EpssItem>> consumerRecords) throws ProcessingException {
try (QueryManager qm = new QueryManager()) {
LOGGER.debug("Synchronizing batch of %s mirrored EPSS records.".formatted(consumerRecords.size()));
List<Epss> epssList = consumerRecords.stream()
.map(ConsumerRecord::value)
.map(EpssModelConverter::convert)
.toList();
if (!epssList.isEmpty()) {
qm.synchronizeAllEpss(epssList);
}
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,8 @@ public void contextInitialized(final ServletContextEvent event) {
KafkaTopics.NEW_VULNERABILITY, new VulnerabilityMirrorProcessor());
PROCESSOR_MANAGER.registerProcessor(RepositoryMetaResultProcessor.PROCESSOR_NAME,
KafkaTopics.REPO_META_ANALYSIS_RESULT, new RepositoryMetaResultProcessor());
PROCESSOR_MANAGER.registerBatchProcessor(EpssMirrorProcessor.PROCESSOR_NAME,
KafkaTopics.NEW_EPSS, new EpssMirrorProcessor());

PROCESSOR_MANAGER.startAll();
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -329,10 +329,6 @@ private Vulnerability syncVulnerability(final QueryManager qm, final Vulnerabili

differ.applyIfChanged("vulnerableVersions", Vulnerability::getVulnerableVersions, existingVuln::setVulnerableVersions);
differ.applyIfChanged("patchedVersions", Vulnerability::getPatchedVersions, existingVuln::setPatchedVersions);
// EPSS is an additional enrichment that no scanner currently provides.
// We don't want EPSS scores of CVEs to be purged just because the CVE information came from e.g. OSS Index.
differ.applyIfNonNullAndChanged("epssScore", Vulnerability::getEpssScore, existingVuln::setEpssScore);
differ.applyIfNonNullAndChanged("epssPercentile", Vulnerability::getEpssPercentile, existingVuln::setEpssPercentile);

if (!differ.getDiffs().isEmpty()) {
// TODO: Send a notification?
Expand Down
86 changes: 86 additions & 0 deletions src/main/java/org/dependencytrack/model/Epss.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
/*
* This file is part of Dependency-Track.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
* Copyright (c) OWASP Foundation. All Rights Reserved.
*/
package org.dependencytrack.model;

import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonInclude;

import javax.jdo.annotations.Column;
import javax.jdo.annotations.IdGeneratorStrategy;
import javax.jdo.annotations.PersistenceCapable;
import javax.jdo.annotations.Persistent;
import javax.jdo.annotations.PrimaryKey;
import javax.validation.constraints.NotBlank;
import java.io.Serializable;
import java.math.BigDecimal;

@PersistenceCapable
@JsonInclude(JsonInclude.Include.NON_NULL)
public class Epss implements Serializable {

@PrimaryKey
@Persistent(valueStrategy = IdGeneratorStrategy.NATIVE)
@JsonIgnore
private long id;

@Persistent
@Column(name = "CVE", allowsNull = "false")
@NotBlank
private String cve;

@Persistent
@Column(name = "SCORE", scale = 5)
private BigDecimal score;

@Persistent
@Column(name = "PERCENTILE", scale = 5)
private BigDecimal percentile;

public long getId() {
return id;
}

public void setId(long id) {
this.id = id;
}

public String getCve() {
return cve;
}

public void setCve(String cve) {
this.cve = cve;
}

public BigDecimal getScore() {
return score;
}

public void setScore(BigDecimal score) {
this.score = score;
}

public BigDecimal getPercentile() {
return percentile;
}

public void setPercentile(BigDecimal percentile) {
this.percentile = percentile;
}
}
5 changes: 3 additions & 2 deletions src/main/java/org/dependencytrack/model/Finding.java
Original file line number Diff line number Diff line change
Expand Up @@ -73,8 +73,8 @@ public class Finding implements Serializable {
"\"VULNERABILITY\".\"OWASPRRLIKELIHOODSCORE\"," +
"\"VULNERABILITY\".\"OWASPRRTECHNICALIMPACTSCORE\"," +
"\"VULNERABILITY\".\"OWASPRRBUSINESSIMPACTSCORE\"," +
"\"VULNERABILITY\".\"EPSSSCORE\"," +
"\"VULNERABILITY\".\"EPSSPERCENTILE\"," +
"\"EPSS\".\"SCORE\"," +
"\"EPSS\".\"PERCENTILE\"," +
"\"VULNERABILITY\".\"CWES\"," +
"\"FINDINGATTRIBUTION\".\"ANALYZERIDENTITY\"," +
"\"FINDINGATTRIBUTION\".\"ATTRIBUTED_ON\"," +
Expand All @@ -85,6 +85,7 @@ public class Finding implements Serializable {
"FROM \"COMPONENT\" " +
"INNER JOIN \"COMPONENTS_VULNERABILITIES\" ON (\"COMPONENT\".\"ID\" = \"COMPONENTS_VULNERABILITIES\".\"COMPONENT_ID\") " +
"INNER JOIN \"VULNERABILITY\" ON (\"COMPONENTS_VULNERABILITIES\".\"VULNERABILITY_ID\" = \"VULNERABILITY\".\"ID\") " +
"LEFT JOIN \"EPSS\" ON (\"VULNERABILITY\".\"VULNID\" = \"EPSS\".\"CVE\") " +
"INNER JOIN \"FINDINGATTRIBUTION\" ON (\"COMPONENT\".\"ID\" = \"FINDINGATTRIBUTION\".\"COMPONENT_ID\") AND (\"VULNERABILITY\".\"ID\" = \"FINDINGATTRIBUTION\".\"VULNERABILITY_ID\")" +
"LEFT JOIN \"ANALYSIS\" ON (\"COMPONENT\".\"ID\" = \"ANALYSIS\".\"COMPONENT_ID\") AND (\"VULNERABILITY\".\"ID\" = \"ANALYSIS\".\"VULNERABILITY_ID\") AND (\"COMPONENT\".\"PROJECT_ID\" = \"ANALYSIS\".\"PROJECT_ID\") " +
"WHERE \"COMPONENT\".\"PROJECT_ID\" = ?";
Expand Down
38 changes: 14 additions & 24 deletions src/main/java/org/dependencytrack/model/Vulnerability.java
Original file line number Diff line number Diff line change
Expand Up @@ -304,14 +304,6 @@ public static boolean isKnownSource(String source) {
@JsonIgnore
private List<VulnerableSoftware> vulnerableSoftware;

@Persistent
@Column(name = "EPSSSCORE", scale = 5)
private BigDecimal epssScore;

@Persistent
@Column(name = "EPSSPERCENTILE", scale = 5)
private BigDecimal epssPercentile;

@Persistent(mappedBy = "vulnerabilities")
@Order(extensions = @Extension(vendorName = "datanucleus", key = "list-ordering", value = "id ASC"))
private List<Component> components;
Expand All @@ -326,6 +318,8 @@ public static boolean isKnownSource(String source) {
@NotNull
private UUID uuid;

private transient Epss epss;

private transient int affectedProjectCount;

private transient FindingAttribution findingAttribution;
Expand Down Expand Up @@ -632,22 +626,6 @@ public void setCvssV3Vector(String cvssV3Vector) {
this.cvssV3Vector = cvssV3Vector;
}

public BigDecimal getEpssScore() {
return epssScore;
}

public void setEpssScore(BigDecimal epssScore) {
this.epssScore = epssScore;
}

public BigDecimal getEpssPercentile() {
return epssPercentile;
}

public void setEpssPercentile(BigDecimal epssPercentile) {
this.epssPercentile = epssPercentile;
}

public List<VulnerableSoftware> getVulnerableSoftware() {
return vulnerableSoftware;
}
Expand Down Expand Up @@ -743,4 +721,16 @@ public String getOwaspRRVector() {
public void setOwaspRRVector(String owaspRRVector) {
this.owaspRRVector = owaspRRVector;
}

public void setEpss(Epss epss) {
this.epss = epss;
}

public BigDecimal getEpssScore() {
return epss != null ? epss.getScore() : null;
}

public BigDecimal getEpssPercentile() {
return epss != null ? epss.getPercentile() : null;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,6 @@ public static org.dependencytrack.proto.policy.v1.Vulnerability mapToProto(final
maybeSet(vuln::getOwaspRRVector, protoBuilder::setOwaspRrVector);
maybeSet(asDouble(vuln.getEpssScore()), protoBuilder::setEpssScore);
maybeSet(asDouble(vuln.getEpssPercentile()), protoBuilder::setEpssPercentile);

return protoBuilder.build();
}

Expand Down
Loading

0 comments on commit fee43d8

Please sign in to comment.