Implement BOM upload storage plugin

Signed-off-by: nscuro <[email protected]>
DependencyTrack · Aug 5, 2024 · 5babe60 · 5babe60
1 parent 20e81d1
commit 5babe60
Show file tree

Hide file tree

Showing 28 changed files with 1,069 additions and 374 deletions.
diff --git a/pom.xml b/pom.xml
@@ -509,6 +509,12 @@
             <version>${lib.json-unit.version}</version>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.testcontainers</groupId>
+            <artifactId>minio</artifactId>
+            <version>${lib.testcontainers.version}</version>
+            <scope>test</scope>
+        </dependency>
         <dependency>
             <groupId>org.testcontainers</groupId>
             <artifactId>redpanda</artifactId>

diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/BomUploadProcessor.java b/src/main/java/org/dependencytrack/event/kafka/processor/BomUploadProcessor.java
@@ -23,7 +23,6 @@
 import alpine.event.framework.ChainableEvent;
 import alpine.event.framework.Event;
 import alpine.event.framework.EventService;
-import alpine.model.ConfigProperty;
 import alpine.notification.Notification;
 import alpine.notification.NotificationLevel;
 import org.apache.commons.collections4.MultiValuedMap;
@@ -64,8 +63,9 @@
 import org.dependencytrack.notification.vo.BomConsumedOrProcessed;
 import org.dependencytrack.notification.vo.BomProcessingFailed;
 import org.dependencytrack.persistence.QueryManager;
+import org.dependencytrack.plugin.PluginManager;
 import org.dependencytrack.proto.event.v1alpha1.BomUploadedEvent;
-import org.dependencytrack.storage.BomUploadStorageProvider;
+import org.dependencytrack.storage.BomUploadStorage;
 import org.dependencytrack.util.InternalComponentIdentifier;
 import org.json.JSONArray;
 import org.slf4j.MDC;
@@ -107,7 +107,6 @@
 import static org.dependencytrack.common.MdcKeys.MDC_PROJECT_VERSION;
 import static org.dependencytrack.event.kafka.componentmeta.RepoMetaConstants.SUPPORTED_PACKAGE_URLS_FOR_INTEGRITY_CHECK;
 import static org.dependencytrack.event.kafka.componentmeta.RepoMetaConstants.TIME_SPAN;
-import static org.dependencytrack.model.ConfigPropertyConstants.BOM_UPLOAD_STORAGE_PROVIDER;
 import static org.dependencytrack.parser.cyclonedx.util.ModelConverter.convertComponents;
 import static org.dependencytrack.parser.cyclonedx.util.ModelConverter.convertDependencyGraph;
 import static org.dependencytrack.parser.cyclonedx.util.ModelConverter.convertServices;
@@ -157,6 +156,7 @@ private Context(final UUID token, final BomUploadedEvent.Project project) {
     public BomUploadProcessor() {
         this(new KafkaEventDispatcher(), Config.getInstance().getPropertyAsBoolean(ConfigKey.TMP_DELAY_BOM_PROCESSED_NOTIFICATION));
     }
+
     BomUploadProcessor(final KafkaEventDispatcher kafkaEventDispatcher, final boolean delayBomProcessedNotification) {
         this.kafkaEventDispatcher = kafkaEventDispatcher;
         this.delayBomProcessedNotification = delayBomProcessedNotification;
@@ -166,25 +166,14 @@ public BomUploadProcessor() {
     public void process(final ConsumerRecord<UUID, BomUploadedEvent> record) throws ProcessingException {
         final BomUploadedEvent event = record.value();
 
-        final BomUploadStorageProvider storageProvider;
-        try (final var qm = new QueryManager()) {
-            final ConfigProperty storageProviderProperty = qm.getConfigProperty(
-                    BOM_UPLOAD_STORAGE_PROVIDER.getGroupName(),
-                    BOM_UPLOAD_STORAGE_PROVIDER.getPropertyName()
-            );
-            final String storageProviderClassName = storageProviderProperty != null
-                    ? storageProviderProperty.getPropertyValue()
-                    : BOM_UPLOAD_STORAGE_PROVIDER.getDefaultPropertyValue();
-            storageProvider = BomUploadStorageProvider.getForClassName(storageProviderClassName);
-        }
-
         final var ctx = new Context(UUID.fromString(event.getToken()), event.getProject());
         try (var ignoredMdcProjectUuid = MDC.putCloseable(MDC_PROJECT_UUID, ctx.project.getUuid().toString());
              var ignoredMdcProjectName = MDC.putCloseable(MDC_PROJECT_NAME, ctx.project.getName());
              var ignoredMdcProjectVersion = MDC.putCloseable(MDC_PROJECT_VERSION, ctx.project.getVersion());
-             var ignoredMdcBomUploadToken = MDC.putCloseable(MDC_BOM_UPLOAD_TOKEN, ctx.token.toString())) {
+             var ignoredMdcBomUploadToken = MDC.putCloseable(MDC_BOM_UPLOAD_TOKEN, ctx.token.toString());
+             final BomUploadStorage storageProvider = PluginManager.getInstance().getExtension(BomUploadStorage.class)) {
             processEvent(ctx, storageProvider);
-        } finally {
+
             try {
                 storageProvider.deleteBomByToken(ctx.token);
             } catch (IOException | RuntimeException e) {
@@ -193,7 +182,7 @@ public void process(final ConsumerRecord<UUID, BomUploadedEvent> record) throws
         }
     }
 
-    private void processEvent(final Context ctx, final BomUploadStorageProvider storageProvider) {
+    private void processEvent(final Context ctx, final BomUploadStorage storageProvider) {
         startBomConsumptionWorkflowStep(ctx);
 
         final ConsumedBom consumedBom;

diff --git a/src/main/java/org/dependencytrack/model/ConfigPropertyConstants.java b/src/main/java/org/dependencytrack/model/ConfigPropertyConstants.java
@@ -20,13 +20,10 @@
 
 import alpine.model.IConfigProperty;
 import alpine.model.IConfigProperty.PropertyType;
-import com.github.luben.zstd.Zstd;
 import org.apache.commons.lang3.SystemUtils;
-import org.dependencytrack.storage.DatabaseBomUploadStorageProvider;
 
 import java.util.Arrays;
 import java.util.UUID;
-import java.util.concurrent.TimeUnit;
 
 public enum ConfigPropertyConstants {
 
@@ -71,9 +68,6 @@ public enum ConfigPropertyConstants {
     VULNERABILITY_SOURCE_EPSS_FEEDS_URL("vuln-source", "epss.feeds.url", "https://epss.cyentia.com", PropertyType.URL, "A base URL pointing to the hostname and path of the EPSS feeds", ConfigPropertyAccessMode.READ_WRITE),
     ACCEPT_ARTIFACT_CYCLONEDX("artifact", "cyclonedx.enabled", "true", PropertyType.BOOLEAN, "Flag to enable/disable the systems ability to accept CycloneDX uploads", ConfigPropertyAccessMode.READ_WRITE),
     BOM_VALIDATION_ENABLED("artifact", "bom.validation.enabled", "true", PropertyType.BOOLEAN, "Flag to control bom validation", ConfigPropertyAccessMode.READ_WRITE),
-    BOM_UPLOAD_STORAGE_PROVIDER("artifact", "bom.upload.storage.provider", DatabaseBomUploadStorageProvider.class.getName(), PropertyType.STRING, "Class of the BOM upload storage provider", ConfigPropertyAccessMode.READ_WRITE),
-    BOM_UPLOAD_STORAGE_COMPRESSION_LEVEL("artifact", "bom.upload.storage.compression.level", String.valueOf(Zstd.defaultCompressionLevel()), PropertyType.INTEGER, "Compression level to use for storage of uploaded BOMs", ConfigPropertyAccessMode.READ_WRITE),
-    BOM_UPLOAD_STORAGE_RETENTION_MS("artifact", "bom.upload.storage.retention.ms", String.valueOf(TimeUnit.HOURS.toMillis(1)), PropertyType.INTEGER, "Maximum storage retention duration for uploaded BOMs in milliseconds", ConfigPropertyAccessMode.READ_WRITE),
     FORTIFY_SSC_ENABLED("integrations", "fortify.ssc.enabled", "false", PropertyType.BOOLEAN, "Flag to enable/disable Fortify SSC integration", ConfigPropertyAccessMode.READ_WRITE),
     FORTIFY_SSC_SYNC_CADENCE("integrations", "fortify.ssc.sync.cadence", "60", PropertyType.INTEGER, "The cadence (in minutes) to upload to Fortify SSC", ConfigPropertyAccessMode.READ_WRITE),
     FORTIFY_SSC_URL("integrations", "fortify.ssc.url", null, PropertyType.URL, "Base URL to Fortify SSC", ConfigPropertyAccessMode.READ_WRITE),

diff --git a/src/main/java/org/dependencytrack/persistence/jdbi/BomDao.java b/src/main/java/org/dependencytrack/persistence/jdbi/BomDao.java
@@ -34,6 +34,10 @@ public interface BomDao {
     @SqlUpdate("""
             INSERT INTO "BOM_UPLOAD" ("TOKEN", "UPLOADED_AT", "BOM")
             VALUES (:token, NOW(), :bomBytes)
+                ON CONFLICT ("TOKEN")
+                DO UPDATE
+               SET "UPLOADED_AT" = NOW()
+                 , "BOM" = :bomBytes
             """)
     void createUpload(@Bind UUID token, @Bind byte[] bomBytes);
 

diff --git a/src/main/java/org/dependencytrack/resources/v1/AbstractConfigPropertyResource.java b/src/main/java/org/dependencytrack/resources/v1/AbstractConfigPropertyResource.java
@@ -25,11 +25,9 @@
 import alpine.security.crypto.DataEncryption;
 import alpine.server.resources.AlpineResource;
 import jakarta.ws.rs.core.Response;
-import com.github.luben.zstd.Zstd;
 import org.dependencytrack.model.ConfigPropertyAccessMode;
 import org.dependencytrack.model.ConfigPropertyConstants;
 import org.dependencytrack.persistence.QueryManager;
-import org.dependencytrack.storage.BomUploadStorageProvider;
 import org.owasp.security.logging.SecurityMarkers;
 
 import java.math.BigDecimal;
@@ -67,24 +65,6 @@ private Response updatePropertyValueInternal(IConfigProperty json, IConfigProper
                     .build();
         }
 
-        if (wellKnownProperty == ConfigPropertyConstants.BOM_UPLOAD_STORAGE_PROVIDER
-                && !BomUploadStorageProvider.exists(json.getPropertyValue())) {
-            return Response
-                    .status(Response.Status.BAD_REQUEST)
-                    .entity("%s is not a known storage provider".formatted(json.getPropertyValue()))
-                    .build();
-        } else if (wellKnownProperty == ConfigPropertyConstants.BOM_UPLOAD_STORAGE_COMPRESSION_LEVEL
-                && json.getPropertyValue() != null) {
-            final int compressionLevel = Integer.parseInt(json.getPropertyValue());
-            if (compressionLevel < 1 || compressionLevel > Zstd.maxCompressionLevel()) {
-                return Response
-                        .status(Response.Status.BAD_REQUEST)
-                        .entity("Compression level %d is out of the valid [1..%d] range"
-                                .formatted(compressionLevel, Zstd.maxCompressionLevel()))
-                        .build();
-            }
-        }
-
         if (property.getPropertyType() == IConfigProperty.PropertyType.BOOLEAN) {
             boolean propertyValue = BooleanUtil.valueOf(json.getPropertyValue());
             if (ConfigPropertyConstants.CUSTOM_RISK_SCORE_HISTORY_ENABLED.getPropertyName().equals(json.getPropertyName())) {

diff --git a/src/main/java/org/dependencytrack/resources/v1/BomResource.java b/src/main/java/org/dependencytrack/resources/v1/BomResource.java
@@ -19,7 +19,6 @@
 package org.dependencytrack.resources.v1;
 
 import alpine.common.logging.Logger;
-import alpine.model.ConfigProperty;
 import alpine.server.auth.PermissionRequired;
 import alpine.server.resources.AlpineResource;
 import io.swagger.v3.oas.annotations.Operation;
@@ -61,12 +60,13 @@
 import org.dependencytrack.parser.cyclonedx.CycloneDxValidator;
 import org.dependencytrack.parser.cyclonedx.InvalidBomException;
 import org.dependencytrack.persistence.QueryManager;
+import org.dependencytrack.plugin.PluginManager;
 import org.dependencytrack.resources.v1.problems.InvalidBomProblemDetails;
 import org.dependencytrack.resources.v1.problems.ProblemDetails;
 import org.dependencytrack.resources.v1.vo.BomSubmitRequest;
 import org.dependencytrack.resources.v1.vo.BomUploadResponse;
 import org.dependencytrack.resources.v1.vo.IsTokenBeingProcessedResponse;
-import org.dependencytrack.storage.BomUploadStorageProvider;
+import org.dependencytrack.storage.BomUploadStorage;
 import org.glassfish.jersey.media.multipart.BodyPartEntity;
 import org.glassfish.jersey.media.multipart.FormDataBodyPart;
 import org.glassfish.jersey.media.multipart.FormDataParam;
@@ -82,8 +82,6 @@
 import java.util.concurrent.atomic.AtomicBoolean;
 
 import static java.util.function.Predicate.not;
-import static org.dependencytrack.model.ConfigPropertyConstants.BOM_UPLOAD_STORAGE_COMPRESSION_LEVEL;
-import static org.dependencytrack.model.ConfigPropertyConstants.BOM_UPLOAD_STORAGE_PROVIDER;
 import static org.dependencytrack.model.ConfigPropertyConstants.BOM_VALIDATION_ENABLED;
 
 /**
@@ -484,7 +482,7 @@ private Response process(QueryManager qm, Project project, String encodedBomData
                  final var decodedInputStream = Base64.getDecoder().wrap(encodedInputStream);
                  final var byteOrderMarkInputStream = new BOMInputStream(decodedInputStream)) {
                 final byte[] bomBytes = IOUtils.toByteArray(byteOrderMarkInputStream);
-                validateAndStoreBom(qm, bomUploadEvent.getChainIdentifier(), bomBytes);
+                validateAndStoreBom(bomUploadEvent.getChainIdentifier(), bomBytes);
             } catch (IOException e) {
                 LOGGER.error("An unexpected error occurred while validating or storing a BOM uploaded to project: " + project.getUuid(), e);
                 return Response.status(Response.Status.INTERNAL_SERVER_ERROR).build();
@@ -516,7 +514,7 @@ private Response process(QueryManager qm, Project project, List<FormDataBodyPart
                 try (final var inputStream = bodyPartEntity.getInputStream();
                      final var byteOrderMarkInputStream = new BOMInputStream(inputStream)) {
                     final byte[] bomBytes = IOUtils.toByteArray(byteOrderMarkInputStream);
-                    validateAndStoreBom(qm, bomUploadEvent.getChainIdentifier(), bomBytes);
+                    validateAndStoreBom(bomUploadEvent.getChainIdentifier(), bomBytes);
                 } catch (IOException e) {
                     LOGGER.error("An unexpected error occurred while validating or storing a BOM uploaded to project: " + project.getUuid(), e);
                     return Response.status(Response.Status.INTERNAL_SERVER_ERROR).build();
@@ -535,27 +533,12 @@ private Response process(QueryManager qm, Project project, List<FormDataBodyPart
         return Response.ok().build();
     }
 
-    private void validateAndStoreBom(final QueryManager qm, final UUID token, final byte[] bomBytes) throws IOException {
+    private void validateAndStoreBom(final UUID token, final byte[] bomBytes) throws IOException {
         validate(bomBytes);
 
-        final ConfigProperty storageProviderProperty = qm.getConfigProperty(
-                BOM_UPLOAD_STORAGE_PROVIDER.getGroupName(),
-                BOM_UPLOAD_STORAGE_PROVIDER.getPropertyName()
-        );
-        final String storageProviderClassName = storageProviderProperty != null
-                ? storageProviderProperty.getPropertyValue()
-                : BOM_UPLOAD_STORAGE_PROVIDER.getDefaultPropertyValue();
-        final var storageProvider = BomUploadStorageProvider.getForClassName(storageProviderClassName);
-
-        final ConfigProperty compressionLevelProperty = qm.getConfigProperty(
-                BOM_UPLOAD_STORAGE_COMPRESSION_LEVEL.getGroupName(),
-                BOM_UPLOAD_STORAGE_COMPRESSION_LEVEL.getPropertyName()
-        );
-        final int compressionLevel = compressionLevelProperty != null
-                ? Integer.parseInt(compressionLevelProperty.getPropertyValue())
-                : Integer.parseInt(BOM_UPLOAD_STORAGE_COMPRESSION_LEVEL.getDefaultPropertyValue());
-
-        storageProvider.storeBomCompressed(token, bomBytes, compressionLevel);
+        try (final BomUploadStorage storageProvider = PluginManager.getInstance().getExtension(BomUploadStorage.class)) {
+            storageProvider.storeBomCompressed(token, bomBytes, /* TODO: Make configurable */ 3);
+        }
     }
 
     static void validate(final byte[] bomBytes) {

diff --git a/...ack/storage/BomUploadStorageProvider.java → ...ndencytrack/storage/BomUploadStorage.java b/...ack/storage/BomUploadStorageProvider.java → ...ndencytrack/storage/BomUploadStorage.java
@@ -19,31 +19,57 @@
 package org.dependencytrack.storage;
 
 import com.github.luben.zstd.Zstd;
+import org.dependencytrack.plugin.api.ExtensionPoint;
 
 import java.io.IOException;
 import java.time.Duration;
-import java.util.NoSuchElementException;
-import java.util.ServiceLoader;
 import java.util.UUID;
 
 /**
  * @since 5.6.0
  */
-public interface BomUploadStorageProvider {
+public interface BomUploadStorage extends ExtensionPoint {
 
+    /**
+     * @param token The token to store the BOM for.
+     * @param bom   The BOM to store.
+     * @throws IOException When storing the BOM failed.
+     */
     void storeBom(final UUID token, final byte[] bom) throws IOException;
 
+    /**
+     * @param token The token to get the BOM for.
+     * @return The BOM, or {@code null} when no BOM was found.
+     * @throws IOException When getting the BOM failed.
+     */
     byte[] getBomByToken(final UUID token) throws IOException;
 
+    /**
+     * @param token The token to delete the BOM for.
+     * @return {@code true} when the BOM was deleted, otherwise {@code false}.
+     * @throws IOException When deleting the BOM failed.
+     */
     boolean deleteBomByToken(final UUID token) throws IOException;
 
     int deleteBomsForRetentionDuration(final Duration duration) throws IOException;
 
+    /**
+     * @param token            The token to store the BOM for.
+     * @param bom              The BOM to store.
+     * @param compressionLevel The compression level to use.
+     * @throws IOException When storing the BOM failed.
+     * @see #storeBom(UUID, byte[])
+     */
     default void storeBomCompressed(final UUID token, final byte[] bom, final int compressionLevel) throws IOException {
         final byte[] compressedBom = Zstd.compress(bom, compressionLevel);
         storeBom(token, compressedBom);
     }
 
+    /**
+     * @param token The token to get the BOM for.
+     * @return The BOM, or {@code null} when no BOM was found.
+     * @throws IOException When getting the BOM failed.
+     */
     default byte[] getDecompressedBomByToken(final UUID token) throws IOException {
         final byte[] compressedBom = getBomByToken(token);
         if (compressedBom == null) {
@@ -58,18 +84,4 @@ default byte[] getDecompressedBomByToken(final UUID token) throws IOException {
         return Zstd.decompress(compressedBom, (int) decompressedSize);
     }
 
-    static BomUploadStorageProvider getForClassName(final String providerClassName) {
-        final var serviceLoader = ServiceLoader.load(BomUploadStorageProvider.class);
-        return serviceLoader.stream()
-                .filter(provider -> provider.type().getName().equals(providerClassName))
-                .findFirst()
-                .map(ServiceLoader.Provider::get)
-                .orElseThrow(() -> new NoSuchElementException("%s is not a known storage provider".formatted(providerClassName)));
-    }
-
-    static boolean exists(final String providerClassName) {
-        final var serviceLoader = ServiceLoader.load(BomUploadStorageProvider.class);
-        return serviceLoader.stream().anyMatch(provider -> provider.type().getName().equals(providerClassName));
-    }
-
 }
diff --git a/src/main/java/org/dependencytrack/storage/BomUploadStorageExtensionPointMetadata.java b/src/main/java/org/dependencytrack/storage/BomUploadStorageExtensionPointMetadata.java
@@ -0,0 +1,45 @@
+/*
+ * This file is part of Dependency-Track.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright (c) OWASP Foundation. All Rights Reserved.
+ */
+package org.dependencytrack.storage;
+
+import org.dependencytrack.plugin.api.ExtensionPointMetadata;
+
+/**
+ * @since 5.6.0
+ */
+public class BomUploadStorageExtensionPointMetadata implements ExtensionPointMetadata<BomUploadStorage> {
+
+    static final String EXTENSION_POINT_NAME = "bom.upload.storage";
+
+    @Override
+    public String name() {
+        return EXTENSION_POINT_NAME;
+    }
+
+    @Override
+    public boolean required() {
+        return true;
+    }
+
+    @Override
+    public Class<BomUploadStorage> extensionPointClass() {
+        return BomUploadStorage.class;
+    }
+
+}