diff --git a/.idea/runConfigurations/Jetty.xml b/.idea/runConfigurations/Jetty.xml index a947494be..b98cd238b 100644 --- a/.idea/runConfigurations/Jetty.xml +++ b/.idea/runConfigurations/Jetty.xml @@ -7,8 +7,6 @@ - - diff --git a/.idea/runConfigurations/Jetty_w__Dev_Services.xml b/.idea/runConfigurations/Jetty_w__Dev_Services.xml new file mode 100644 index 000000000..37abe9060 --- /dev/null +++ b/.idea/runConfigurations/Jetty_w__Dev_Services.xml @@ -0,0 +1,52 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/pom.xml b/pom.xml index 436f7f144..6445b9f83 100644 --- a/pom.xml +++ b/pom.xml @@ -646,6 +646,7 @@ org/cyclonedx/proto/**/* + org/dependencytrack/dev/**/* org/dependencytrack/proto/**/* @@ -755,6 +756,47 @@ ${project.build.finalName}-apiserver + + dev-services + + + org.testcontainers + postgresql + ${lib.testcontainers.version} + compile + + + org.testcontainers + redpanda + ${lib.testcontainers.version} + compile + + + + junit + junit + ${lib.junit.version} + compile + + + + + + org.eclipse.jetty + jetty-maven-plugin + ${plugin.jetty.version} + + + true + + + + + + diff --git a/src/main/java/org/dependencytrack/dev/DevServicesInitializer.java b/src/main/java/org/dependencytrack/dev/DevServicesInitializer.java new file mode 100644 index 000000000..94e0b814e --- /dev/null +++ b/src/main/java/org/dependencytrack/dev/DevServicesInitializer.java @@ -0,0 +1,170 @@ +/* + * This file is part of Dependency-Track. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) OWASP Foundation. All Rights Reserved. + */ +package org.dependencytrack.dev; + +import alpine.Config; +import alpine.common.logging.Logger; +import org.apache.kafka.clients.admin.AdminClient; +import org.apache.kafka.clients.admin.NewTopic; +import org.dependencytrack.event.kafka.KafkaTopics; + +import javax.servlet.ServletContextEvent; +import javax.servlet.ServletContextListener; +import java.lang.reflect.Constructor; +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; + +import static alpine.Config.AlpineKey.DATABASE_PASSWORD; +import static alpine.Config.AlpineKey.DATABASE_URL; +import static alpine.Config.AlpineKey.DATABASE_USERNAME; +import static org.apache.kafka.clients.admin.AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG; +import static org.apache.kafka.common.config.TopicConfig.CLEANUP_POLICY_COMPACT; +import static org.apache.kafka.common.config.TopicConfig.CLEANUP_POLICY_CONFIG; +import static org.dependencytrack.common.ConfigKey.KAFKA_BOOTSTRAP_SERVERS; + +/** + * @since 5.5.0 + */ +public class DevServicesInitializer implements ServletContextListener { + + private static final Logger LOGGER = Logger.getLogger(DevServicesInitializer.class); + private static final String POSTGRES_IMAGE = "postgres:16-alpine"; + private static final String REDPANDA_IMAGE = "docker.redpanda.com/vectorized/redpanda:v24.1.7"; + + private AutoCloseable postgresContainer; + private AutoCloseable redpandaContainer; + + @Override + public void contextInitialized(final ServletContextEvent event) { + if (!"true".equals(System.getProperty("dev.services.enabled"))) { + return; + } + + final String postgresJdbcUrl; + final String postgresUsername; + final String postgresPassword; + final String redpandaBootstrapServers; + try { + final Class> startablesClass = Class.forName("org.testcontainers.lifecycle.Startables"); + final Method deepStartMethod = startablesClass.getDeclaredMethod("deepStart", Collection.class); + + final Class> postgresContainerClass = Class.forName("org.testcontainers.containers.PostgreSQLContainer"); + final Constructor> postgresContainerConstructor = postgresContainerClass.getDeclaredConstructor(String.class); + postgresContainer = (AutoCloseable) postgresContainerConstructor.newInstance(POSTGRES_IMAGE); + + final Class> redpandaContainerClass = Class.forName("org.testcontainers.redpanda.RedpandaContainer"); + final Constructor> redpandaContainerConstructor = redpandaContainerClass.getDeclaredConstructor(String.class); + redpandaContainer = (AutoCloseable) redpandaContainerConstructor.newInstance(REDPANDA_IMAGE); + + LOGGER.info("Starting PostgreSQL and Redpanda containers"); + final var deepStartFuture = (CompletableFuture>) deepStartMethod.invoke(null, List.of(postgresContainer, redpandaContainer)); + deepStartFuture.join(); + + postgresJdbcUrl = (String) postgresContainerClass.getDeclaredMethod("getJdbcUrl").invoke(postgresContainer); + postgresUsername = (String) postgresContainerClass.getDeclaredMethod("getUsername").invoke(postgresContainer); + postgresPassword = (String) postgresContainerClass.getDeclaredMethod("getPassword").invoke(postgresContainer); + redpandaBootstrapServers = (String) redpandaContainerClass.getDeclaredMethod("getBootstrapServers").invoke(redpandaContainer); + } catch (Exception e) { + throw new RuntimeException("Failed to launch containers", e); + } + + LOGGER.warn(""" + Containers are not auto-discoverable by other services yet. \ + If interaction with other services is required, please use \ + the Docker Compose setup in the DependencyTrack/hyades repository. \ + Auto-discovery is worked on in https://github.com/DependencyTrack/hyades/issues/1188.\ + """); + + final var configOverrides = new Properties(); + configOverrides.put(DATABASE_URL.getPropertyName(), postgresJdbcUrl); + configOverrides.put(DATABASE_USERNAME.getPropertyName(), postgresUsername); + configOverrides.put(DATABASE_PASSWORD.getPropertyName(), postgresPassword); + configOverrides.put(KAFKA_BOOTSTRAP_SERVERS.getPropertyName(), redpandaBootstrapServers); + + try { + LOGGER.info("Applying config overrides: %s".formatted(configOverrides)); + final Field propertiesField = Config.class.getDeclaredField("properties"); + propertiesField.setAccessible(true); + + final Properties properties = (Properties) propertiesField.get(Config.getInstance()); + properties.putAll(configOverrides); + } catch (Exception e) { + throw new RuntimeException("Failed to update configuration", e); + } + + final var topicsToCreate = new ArrayList<>(List.of( + new NewTopic(KafkaTopics.NEW_EPSS.name(), 1, (short) 1).configs(Map.of(CLEANUP_POLICY_CONFIG, CLEANUP_POLICY_COMPACT)), + new NewTopic(KafkaTopics.NEW_VULNERABILITY.name(), 1, (short) 1).configs(Map.of(CLEANUP_POLICY_CONFIG, CLEANUP_POLICY_COMPACT)), + new NewTopic(KafkaTopics.NOTIFICATION_ANALYZER.name(), 1, (short) 1), + new NewTopic(KafkaTopics.NOTIFICATION_BOM.name(), 1, (short) 1), + new NewTopic(KafkaTopics.NOTIFICATION_CONFIGURATION.name(), 1, (short) 1), + new NewTopic(KafkaTopics.NOTIFICATION_DATASOURCE_MIRRORING.name(), 1, (short) 1), + new NewTopic(KafkaTopics.NOTIFICATION_FILE_SYSTEM.name(), 1, (short) 1), + new NewTopic(KafkaTopics.NOTIFICATION_INTEGRATION.name(), 1, (short) 1), + new NewTopic(KafkaTopics.NOTIFICATION_NEW_VULNERABILITY.name(), 1, (short) 1), + new NewTopic(KafkaTopics.NOTIFICATION_NEW_VULNERABLE_DEPENDENCY.name(), 1, (short) 1), + new NewTopic(KafkaTopics.NOTIFICATION_POLICY_VIOLATION.name(), 1, (short) 1), + new NewTopic(KafkaTopics.NOTIFICATION_PROJECT_AUDIT_CHANGE.name(), 1, (short) 1), + new NewTopic(KafkaTopics.NOTIFICATION_PROJECT_CREATED.name(), 1, (short) 1), + new NewTopic(KafkaTopics.NOTIFICATION_PROJECT_VULN_ANALYSIS_COMPLETE.name(), 1, (short) 1), + new NewTopic(KafkaTopics.NOTIFICATION_REPOSITORY.name(), 1, (short) 1), + new NewTopic(KafkaTopics.NOTIFICATION_VEX.name(), 1, (short) 1), + new NewTopic(KafkaTopics.REPO_META_ANALYSIS_COMMAND.name(), 1, (short) 1), + new NewTopic(KafkaTopics.REPO_META_ANALYSIS_RESULT.name(), 1, (short) 1), + new NewTopic(KafkaTopics.VULN_ANALYSIS_COMMAND.name(), 1, (short) 1), + new NewTopic(KafkaTopics.VULN_ANALYSIS_RESULT.name(), 1, (short) 1), + new NewTopic(KafkaTopics.VULN_ANALYSIS_RESULT_PROCESSED.name(), 1, (short) 1) + )); + + try (final var adminClient = AdminClient.create(Map.of(BOOTSTRAP_SERVERS_CONFIG, redpandaBootstrapServers))) { + LOGGER.info("Creating topics: %s".formatted(topicsToCreate)); + adminClient.createTopics(topicsToCreate).all().get(); + } catch (ExecutionException | InterruptedException e) { + throw new RuntimeException("Failed to create topics", e); + } + } + + @Override + public void contextDestroyed(final ServletContextEvent event) { + if (postgresContainer != null) { + LOGGER.info("Stopping postgres container"); + try { + postgresContainer.close(); + } catch (Exception e) { + throw new RuntimeException("Failed to stop PostgreSQL container", e); + } + } + if (redpandaContainer != null) { + LOGGER.info("Stopping redpanda container"); + try { + redpandaContainer.close(); + } catch (Exception e) { + throw new RuntimeException("Failed to stop Redpanda container", e); + } + } + } + +} diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/api/ProcessorManager.java b/src/main/java/org/dependencytrack/event/kafka/processor/api/ProcessorManager.java index a390f7915..45fb03fba 100644 --- a/src/main/java/org/dependencytrack/event/kafka/processor/api/ProcessorManager.java +++ b/src/main/java/org/dependencytrack/event/kafka/processor/api/ProcessorManager.java @@ -90,7 +90,7 @@ public class ProcessorManager implements AutoCloseable { private final Map managedProcessors = new LinkedHashMap<>(); private final UUID instanceId; private final Config config; - private final AdminClient adminClient; + private AdminClient adminClient; public ProcessorManager() { this(UUID.randomUUID(), Config.getInstance()); @@ -99,7 +99,6 @@ public ProcessorManager() { public ProcessorManager(final UUID instanceId, final Config config) { this.instanceId = instanceId; this.config = config; - this.adminClient = createAdminClient(); } /** @@ -165,8 +164,8 @@ public HealthCheckResponse probeHealth() { ? HealthCheckResponse.Status.UP.name() : HealthCheckResponse.Status.DOWN.name()); if (isProcessorUp - && parallelConsumer instanceof final ParallelEoSStreamProcessor, ?> concreteParallelConsumer - && concreteParallelConsumer.getFailureCause() != null) { + && parallelConsumer instanceof final ParallelEoSStreamProcessor, ?> concreteParallelConsumer + && concreteParallelConsumer.getFailureCause() != null) { responseBuilder.withData("%s_failure_reason".formatted(processorName), concreteParallelConsumer.getFailureCause().getMessage()); } @@ -198,7 +197,7 @@ private void ensureTopicsExist() { final List topicNames = managedProcessors.values().stream().map(ManagedProcessor::topic).toList(); LOGGER.info("Verifying existence of subscribed topics: %s".formatted(topicNames)); - final DescribeTopicsResult topicsResult = adminClient.describeTopics(topicNames, new DescribeTopicsOptions().timeoutMs(3_000)); + final DescribeTopicsResult topicsResult = adminClient().describeTopics(topicNames, new DescribeTopicsOptions().timeoutMs(3_000)); final var exceptionsByTopicName = new HashMap(); for (final Map.Entry> entry : topicsResult.topicNameValues().entrySet()) { final String topicName = entry.getKey(); @@ -225,7 +224,7 @@ private void ensureTopicsExist() { private int getTopicPartitionCount(final String topicName) { LOGGER.debug("Determining partition count of topic %s".formatted(topicName)); - final DescribeTopicsResult topicsResult = adminClient.describeTopics(List.of(topicName), new DescribeTopicsOptions().timeoutMs(3_000)); + final DescribeTopicsResult topicsResult = adminClient().describeTopics(List.of(topicName), new DescribeTopicsOptions().timeoutMs(3_000)); final KafkaFuture topicDescriptionFuture = topicsResult.topicNameValues().get(topicName); try { @@ -343,14 +342,19 @@ private Consumer createConsumer(final String processorName) { return consumer; } - private AdminClient createAdminClient() { + private AdminClient adminClient() { + if (adminClient != null) { + return adminClient; + } + final var adminClientConfig = new HashMap(); adminClientConfig.put(BOOTSTRAP_SERVERS_CONFIG, config.getProperty(KAFKA_BOOTSTRAP_SERVERS)); adminClientConfig.put(CLIENT_ID_CONFIG, "%s-admin-client".formatted(instanceId)); adminClientConfig.putAll(getGlobalTlsConfig()); LOGGER.debug("Creating admin client with options %s".formatted(adminClientConfig)); - return AdminClient.create(adminClientConfig); + adminClient = AdminClient.create(adminClientConfig); + return adminClient; } private Map getGlobalTlsConfig() { diff --git a/src/main/webapp/WEB-INF/web.xml b/src/main/webapp/WEB-INF/web.xml index 9ffe68339..392ed5cb7 100644 --- a/src/main/webapp/WEB-INF/web.xml +++ b/src/main/webapp/WEB-INF/web.xml @@ -22,7 +22,9 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://xmlns.jcp.org/xml/ns/javaee http://xmlns.jcp.org/xml/ns/javaee/web-app_3_1.xsd" version="3.1"> - + + org.dependencytrack.dev.DevServicesInitializer + alpine.server.metrics.MetricsInitializer