added integrity analysis event on apiserver

Signed-off-by: mehab <[email protected]>
DependencyTrack · Oct 2, 2023 · 5b2d915 · 5b2d915
1 parent a0c2364
commit 5b2d915
Show file tree

Hide file tree

Showing 21 changed files with 474 additions and 38 deletions.
diff --git a/src/main/java/org/dependencytrack/event/ComponentRepositoryMetaAnalysisEvent.java b/src/main/java/org/dependencytrack/event/ComponentRepositoryMetaAnalysisEvent.java
@@ -1,21 +1,18 @@
 package org.dependencytrack.event;
 
 import alpine.event.framework.Event;
-import com.github.packageurl.PackageURL;
 import org.dependencytrack.model.Component;
 
-import java.util.Optional;
-
 /**
  * Defines an {@link Event} triggered when requesting a component to be analyzed for meta information.
  *
- * @param purlCoordinates The package URL coordinates of the {@link Component} to analyze
- * @param internal        Whether the {@link Component} is internal
+ * @param purlCoordinates    The package URL coordinates of the {@link Component} to analyze
+ * @param internal           Whether the {@link Component} is internal
+ * @param fetchIntegrityData Whether component hash information needs to be fetched from external api
+ * @param fetchLatestVersion Whether to fetch latest version meta information for a component.
  */
-public record ComponentRepositoryMetaAnalysisEvent(String purlCoordinates, Boolean internal) implements Event {
-
-    public ComponentRepositoryMetaAnalysisEvent(final Component component) {
-        this(Optional.ofNullable(component.getPurlCoordinates()).map(PackageURL::canonicalize).orElse(null), component.isInternal());
-    }
+public record ComponentRepositoryMetaAnalysisEvent(String purlCoordinates, Boolean internal,
+                                                   boolean fetchIntegrityData,
+                                                   boolean fetchLatestVersion) implements Event {
 
 }
diff --git a/src/main/java/org/dependencytrack/event/IntegrityMetaInitializer.java b/src/main/java/org/dependencytrack/event/IntegrityMetaInitializer.java
@@ -67,7 +67,7 @@ private void batchProcessPurls(QueryManager qm) {
         List<String> purls = qm.fetchNextPurlsPage(offset);
         while (!purls.isEmpty()) {
             long cumulativeProcessingTime = System.currentTimeMillis() - startTime;
-            if(isLockToBeExtended(cumulativeProcessingTime, INTEGRITY_META_INITIALIZER_TASK_LOCK)) {
+            if (isLockToBeExtended(cumulativeProcessingTime, INTEGRITY_META_INITIALIZER_TASK_LOCK)) {
                 LockExtender.extendActiveLock(Duration.ofMinutes(5).plus(lockConfiguration.getLockAtLeastFor()), lockConfiguration.getLockAtLeastFor());
             }
             dispatchPurls(qm, purls);
@@ -88,7 +88,7 @@ private void updateIntegrityMetaForPurls(QueryManager qm, List<String> purls) {
     private void dispatchPurls(QueryManager qm, List<String> purls) {
         for (final var purl : purls) {
             ComponentProjection componentProjection = qm.getComponentByPurl(purl);
-            kafkaEventDispatcher.dispatchAsync(new ComponentRepositoryMetaAnalysisEvent(componentProjection.purlCoordinates, componentProjection.internal));
+            kafkaEventDispatcher.dispatchAsync(new ComponentRepositoryMetaAnalysisEvent(componentProjection.purlCoordinates, componentProjection.internal, true, false));
         }
     }
 

diff --git a/src/main/java/org/dependencytrack/event/kafka/KafkaEventConverter.java b/src/main/java/org/dependencytrack/event/kafka/KafkaEventConverter.java
@@ -58,6 +58,8 @@ static KafkaEvent<String, AnalysisCommand> convert(final ComponentRepositoryMeta
 
         final var analysisCommand = AnalysisCommand.newBuilder()
                 .setComponent(componentBuilder)
+                .setFetchIntegrityData(event.fetchIntegrityData())
+                .setFetchLatestVersion(event.fetchLatestVersion())
                 .build();
 
         return new KafkaEvent<>(KafkaTopics.REPO_META_ANALYSIS_COMMAND, event.purlCoordinates(), analysisCommand, null);

diff --git a/src/main/java/org/dependencytrack/event/kafka/componentmeta/AbstractMetaHandler.java b/src/main/java/org/dependencytrack/event/kafka/componentmeta/AbstractMetaHandler.java
@@ -0,0 +1,27 @@
+package org.dependencytrack.event.kafka.componentmeta;
+
+import org.dependencytrack.event.kafka.KafkaEventDispatcher;
+import org.dependencytrack.model.FetchStatus;
+import org.dependencytrack.model.IntegrityMetaComponent;
+import org.dependencytrack.persistence.QueryManager;
+
+import java.time.Instant;
+import java.util.Date;
+
+public abstract class AbstractMetaHandler implements Handler {
+
+    ComponentProjection componentProjection;
+    QueryManager queryManager;
+    KafkaEventDispatcher kafkaEventDispatcher;
+    boolean fetchLatestVersion;
+
+
+    public static IntegrityMetaComponent createIntegrityMetaComponent(String purl) {
+        IntegrityMetaComponent integrityMetaComponent1 = new IntegrityMetaComponent();
+        integrityMetaComponent1.setStatus(FetchStatus.IN_PROGRESS);
+        integrityMetaComponent1.setPurl(purl);
+        integrityMetaComponent1.setLastFetch(Date.from(Instant.now()));
+        return integrityMetaComponent1;
+    }
+
+}
diff --git a/src/main/java/org/dependencytrack/event/kafka/componentmeta/ComponentProjection.java b/src/main/java/org/dependencytrack/event/kafka/componentmeta/ComponentProjection.java
@@ -0,0 +1,4 @@
+package org.dependencytrack.event.kafka.componentmeta;
+
+public record ComponentProjection(String purlCoordinates, Boolean internal, String purl) {
+}
diff --git a/src/main/java/org/dependencytrack/event/kafka/componentmeta/Handler.java b/src/main/java/org/dependencytrack/event/kafka/componentmeta/Handler.java
@@ -0,0 +1,7 @@
+package org.dependencytrack.event.kafka.componentmeta;
+
+import org.dependencytrack.model.IntegrityMetaComponent;
+
+public interface Handler {
+    IntegrityMetaComponent handle();
+}
diff --git a/src/main/java/org/dependencytrack/event/kafka/componentmeta/HandlerFactory.java b/src/main/java/org/dependencytrack/event/kafka/componentmeta/HandlerFactory.java
@@ -0,0 +1,19 @@
+package org.dependencytrack.event.kafka.componentmeta;
+
+import com.github.packageurl.MalformedPackageURLException;
+import com.github.packageurl.PackageURL;
+import org.dependencytrack.event.kafka.KafkaEventDispatcher;
+import org.dependencytrack.persistence.QueryManager;
+
+public class HandlerFactory {
+
+    public static Handler createHandler(ComponentProjection componentProjection, QueryManager queryManager, KafkaEventDispatcher kafkaEventDispatcher, boolean fetchLatestVersion) throws MalformedPackageURLException {
+        PackageURL packageURL = new PackageURL(componentProjection.purl());
+        boolean result = RepoMetaConstants.SUPPORTED_PACKAGE_URLS_FOR_INTEGRITY_CHECK.contains(packageURL.getType());
+        if (result) {
+            return new SupportedMetaHandler(componentProjection, queryManager, kafkaEventDispatcher, fetchLatestVersion);
+        } else {
+            return new UnSupportedMetaHandler(componentProjection, queryManager, kafkaEventDispatcher, fetchLatestVersion);
+        }
+    }
+}
diff --git a/src/main/java/org/dependencytrack/event/kafka/componentmeta/RepoMetaConstants.java b/src/main/java/org/dependencytrack/event/kafka/componentmeta/RepoMetaConstants.java
@@ -0,0 +1,9 @@
+package org.dependencytrack.event.kafka.componentmeta;
+
+import java.util.List;
+
+public class RepoMetaConstants {
+
+    public static final long TIME_SPAN = 60 * 60 * 1000L;
+    public static final List<String> SUPPORTED_PACKAGE_URLS_FOR_INTEGRITY_CHECK =List.of("maven", "npm", "pypi");
+}
diff --git a/src/main/java/org/dependencytrack/event/kafka/componentmeta/SupportedMetaHandler.java b/src/main/java/org/dependencytrack/event/kafka/componentmeta/SupportedMetaHandler.java
@@ -0,0 +1,46 @@
+package org.dependencytrack.event.kafka.componentmeta;
+
+import org.dependencytrack.event.ComponentRepositoryMetaAnalysisEvent;
+import org.dependencytrack.event.kafka.KafkaEventDispatcher;
+import org.dependencytrack.model.FetchStatus;
+import org.dependencytrack.model.IntegrityMetaComponent;
+import org.dependencytrack.persistence.QueryManager;
+
+import java.time.Instant;
+import java.util.Date;
+
+import static org.dependencytrack.event.kafka.componentmeta.RepoMetaConstants.TIME_SPAN;
+
+public class SupportedMetaHandler extends AbstractMetaHandler {
+
+    public SupportedMetaHandler(ComponentProjection componentProjection, QueryManager queryManager, KafkaEventDispatcher kafkaEventDispatcher, boolean fetchLatestVersion) {
+        this.componentProjection = componentProjection;
+        this.kafkaEventDispatcher = kafkaEventDispatcher;
+        this.queryManager = queryManager;
+        this.fetchLatestVersion = fetchLatestVersion;
+    }
+
+    @Override
+    public IntegrityMetaComponent handle() {
+        KafkaEventDispatcher kafkaEventDispatcher = new KafkaEventDispatcher();
+        try (QueryManager queryManager = new QueryManager()) {
+            IntegrityMetaComponent integrityMetaComponent = queryManager.getIntegrityMetaComponent(componentProjection.purl());
+            if (integrityMetaComponent != null) {
+                if (integrityMetaComponent.getStatus() == null || (integrityMetaComponent.getStatus() == FetchStatus.IN_PROGRESS && Date.from(Instant.now()).getTime() - integrityMetaComponent.getLastFetch().getTime() > TIME_SPAN)) {
+                    integrityMetaComponent.setLastFetch(Date.from(Instant.now()));
+                    IntegrityMetaComponent integrityMetaComponent1 = queryManager.updateIntegrityMetaComponent(integrityMetaComponent);
+                    kafkaEventDispatcher.dispatchAsync(new ComponentRepositoryMetaAnalysisEvent(componentProjection.purlCoordinates(), componentProjection.internal(), true, fetchLatestVersion));
+                    return integrityMetaComponent1;
+                } else {
+                    kafkaEventDispatcher.dispatchAsync(new ComponentRepositoryMetaAnalysisEvent(componentProjection.purlCoordinates(), componentProjection.internal(), false, fetchLatestVersion));
+                    return integrityMetaComponent;
+                }
+            } else {
+                IntegrityMetaComponent integrityMetaComponent1 = queryManager.createIntegrityMetaComponent(createIntegrityMetaComponent(componentProjection.purl()));
+                kafkaEventDispatcher.dispatchAsync(new ComponentRepositoryMetaAnalysisEvent(componentProjection.purlCoordinates(), componentProjection.internal(), true, fetchLatestVersion));
+                return integrityMetaComponent1;
+            }
+        }
+    }
+
+}
diff --git a/src/main/java/org/dependencytrack/event/kafka/componentmeta/UnSupportedMetaHandler.java b/src/main/java/org/dependencytrack/event/kafka/componentmeta/UnSupportedMetaHandler.java
@@ -0,0 +1,23 @@
+package org.dependencytrack.event.kafka.componentmeta;
+
+import org.dependencytrack.event.ComponentRepositoryMetaAnalysisEvent;
+import org.dependencytrack.event.kafka.KafkaEventDispatcher;
+import org.dependencytrack.model.IntegrityMetaComponent;
+import org.dependencytrack.persistence.QueryManager;
+
+public class UnSupportedMetaHandler extends AbstractMetaHandler {
+
+    public UnSupportedMetaHandler(ComponentProjection componentProjection, QueryManager queryManager, KafkaEventDispatcher kafkaEventDispatcher,boolean fetchLatestVersion) {
+        this.componentProjection = componentProjection;
+        this.kafkaEventDispatcher = kafkaEventDispatcher;
+        this.queryManager = queryManager;
+        this.fetchLatestVersion = fetchLatestVersion;
+    }
+
+    @Override
+    public IntegrityMetaComponent handle() {
+        KafkaEventDispatcher kafkaEventDispatcher = new KafkaEventDispatcher();
+        kafkaEventDispatcher.dispatchAsync(new ComponentRepositoryMetaAnalysisEvent(componentProjection.purlCoordinates(), componentProjection.internal(), false, fetchLatestVersion));
+        return null;
+    }
+}
diff --git a/src/main/java/org/dependencytrack/model/FetchStatus.java b/src/main/java/org/dependencytrack/model/FetchStatus.java
@@ -1,7 +1,12 @@
 package org.dependencytrack.model;
 
 public enum FetchStatus {
+    //request processed successfully
     PROCESSED,
-    TIMED_OUT,
-    IN_PROGRESS
+    //fetching information for this component is in progress
+    IN_PROGRESS,
+
+    //to be used when information is not available in source of truth so we don't go fetching this repo information again
+    //after first attempt
+    NOT_AVAILABLE
 }
diff --git a/src/main/java/org/dependencytrack/persistence/QueryManager.java b/src/main/java/org/dependencytrack/persistence/QueryManager.java
@@ -1722,4 +1722,8 @@ public List<String> fetchNextPurlsPage(long offset) {
     public void batchUpdateIntegrityMetaComponent(List<IntegrityMetaComponent> purls) {
         getIntegrityMetaQueryManager().batchUpdateIntegrityMetaComponent(purls);
     }
+
+    public IntegrityMetaComponent createIntegrityMetaComponent(IntegrityMetaComponent integrityMetaComponent) {
+        return getComponentQueryManager().createIntegrityMetaComponent(integrityMetaComponent);
+    }
 }
diff --git a/src/main/java/org/dependencytrack/resources/v1/ComponentResource.java b/src/main/java/org/dependencytrack/resources/v1/ComponentResource.java
@@ -18,6 +18,7 @@
  */
 package org.dependencytrack.resources.v1;
 
+import alpine.common.logging.Logger;
 import alpine.event.framework.Event;
 import alpine.persistence.PaginatedResult;
 import alpine.server.auth.PermissionRequired;
@@ -33,10 +34,12 @@
 import io.swagger.annotations.ResponseHeader;
 import org.apache.commons.lang3.StringUtils;
 import org.dependencytrack.auth.Permissions;
-import org.dependencytrack.event.ComponentRepositoryMetaAnalysisEvent;
 import org.dependencytrack.event.ComponentVulnerabilityAnalysisEvent;
 import org.dependencytrack.event.InternalComponentIdentificationEvent;
 import org.dependencytrack.event.kafka.KafkaEventDispatcher;
+import org.dependencytrack.event.kafka.componentmeta.ComponentProjection;
+import org.dependencytrack.event.kafka.componentmeta.Handler;
+import org.dependencytrack.event.kafka.componentmeta.HandlerFactory;
 import org.dependencytrack.model.Component;
 import org.dependencytrack.model.ComponentIdentity;
 import org.dependencytrack.model.License;
@@ -74,6 +77,7 @@
 @Api(value = "component", authorizations = @Authorization(value = "X-Api-Key"))
 public class ComponentResource extends AlpineResource {
 
+    private static final Logger LOGGER = Logger.getLogger(ComponentResource.class);
     private final KafkaEventDispatcher kafkaEventDispatcher = new KafkaEventDispatcher();
 
     @GET
@@ -276,7 +280,7 @@ public Response createComponent(@PathParam("uuid") String uuid, Component jsonCo
             if (project == null) {
                 return Response.status(Response.Status.NOT_FOUND).entity("The project could not be found.").build();
             }
-            if (! qm.hasAccess(super.getPrincipal(), project)) {
+            if (!qm.hasAccess(super.getPrincipal(), project)) {
                 return Response.status(Response.Status.FORBIDDEN).entity("Access to the specified project is forbidden").build();
             }
             final License resolvedLicense = qm.getLicense(jsonComponent.getLicense());
@@ -316,7 +320,15 @@ public Response createComponent(@PathParam("uuid") String uuid, Component jsonCo
             component.setNotes(StringUtils.trimToNull(jsonComponent.getNotes()));
 
             component = qm.createComponent(component, true);
-            kafkaEventDispatcher.dispatchBlocking(new ComponentRepositoryMetaAnalysisEvent(component));
+            ComponentProjection componentProjection =
+                    new ComponentProjection(component.getPurlCoordinates().toString(),
+                            component.isInternal(), component.getPurl().toString());
+            try {
+                Handler repoMetaHandler = HandlerFactory.createHandler(componentProjection, qm, kafkaEventDispatcher, true);
+                repoMetaHandler.handle();
+            } catch (MalformedPackageURLException ex) {
+                LOGGER.warn("Unable to process package url %s".formatted(componentProjection.purl()));
+            }
             final var vulnAnalysisEvent = new ComponentVulnerabilityAnalysisEvent(UUID.randomUUID(), component, VulnerabilityAnalysisLevel.MANUAL_ANALYSIS, true);
             qm.createVulnerabilityScan(VulnerabilityScan.TargetType.COMPONENT, component.getUuid(), vulnAnalysisEvent.token().toString(), 1);
             kafkaEventDispatcher.dispatchBlocking(vulnAnalysisEvent);
@@ -361,7 +373,7 @@ public Response updateComponent(Component jsonComponent) {
         try (QueryManager qm = new QueryManager()) {
             Component component = qm.getObjectByUuid(Component.class, jsonComponent.getUuid());
             if (component != null) {
-                if (! qm.hasAccess(super.getPrincipal(), component.getProject())) {
+                if (!qm.hasAccess(super.getPrincipal(), component.getProject())) {
                     return Response.status(Response.Status.FORBIDDEN).entity("Access to the specified component is forbidden").build();
                 }
                 // Name cannot be empty or null - prevent it
@@ -402,7 +414,16 @@ public Response updateComponent(Component jsonComponent) {
                 component.setNotes(StringUtils.trimToNull(jsonComponent.getNotes()));
 
                 component = qm.updateComponent(component, true);
-                kafkaEventDispatcher.dispatchBlocking(new ComponentRepositoryMetaAnalysisEvent(component));
+                ComponentProjection componentProjection =
+                        new ComponentProjection(component.getPurlCoordinates().toString(),
+                                component.isInternal(), component.getPurl().toString());
+                try {
+
+                    Handler repoMetaHandler = HandlerFactory.createHandler(componentProjection, qm, kafkaEventDispatcher, true);
+                    repoMetaHandler.handle();
+                } catch (MalformedPackageURLException ex) {
+                    LOGGER.warn("Unable to determine package url type for this purl %s".formatted(component.getPurl().getType()), ex);
+                }
                 final var vulnAnalysisEvent = new ComponentVulnerabilityAnalysisEvent(UUID.randomUUID(), component, VulnerabilityAnalysisLevel.MANUAL_ANALYSIS, false);
                 qm.createVulnerabilityScan(VulnerabilityScan.TargetType.COMPONENT, component.getUuid(), vulnAnalysisEvent.token().toString(), 1);
                 kafkaEventDispatcher.dispatchBlocking(vulnAnalysisEvent);
@@ -433,7 +454,7 @@ public Response deleteComponent(
         try (QueryManager qm = new QueryManager()) {
             final Component component = qm.getObjectByUuid(Component.class, uuid, Component.FetchGroup.ALL.name());
             if (component != null) {
-                if (! qm.hasAccess(super.getPrincipal(), component.getProject())) {
+                if (!qm.hasAccess(super.getPrincipal(), component.getProject())) {
                     return Response.status(Response.Status.FORBIDDEN).entity("Access to the specified component is forbidden").build();
                 }
                 qm.recursivelyDelete(component, false);