Migrate all tests to use Postgres testcontainers instead of H2 (#573)

.github/workflows/ci-test.yaml

@@ -43,6 +43,7 @@ jobs:
       - name: Execute unit tests
         env:
           MAVEN_ARGS: "-B --no-transfer-progress"
+          TESTCONTAINERS_REUSE_ENABLE: "true"
         run: |-
           mvn clean
           mvn test -P enhance

pom.xml

@@ -402,13 +402,6 @@
             <scope>test</scope>
         </dependency>
 
-        <dependency>
-            <groupId>com.h2database</groupId>
-            <artifactId>h2</artifactId>
-            <version>2.2.224</version>
-            <scope>test</scope>
-        </dependency>
-
         <!-- Bundle JDBC drivers -->
         <dependency>
             <groupId>org.postgresql</groupId>

src/main/java/org/dependencytrack/model/Component.java

@@ -244,6 +244,7 @@ public enum FetchGroup {
 
     @Persistent
     @Index(name = "COMPONENT_CPE_IDX")
+    @Column(name = "CPE")
     @Size(max = 255)
     //Patterns obtained from https://csrc.nist.gov/schema/cpe/2.3/cpe-naming_2.3.xsd
     @Pattern(regexp = "(cpe:2\\.3:[aho\\*\\-](:(((\\?*|\\*?)([a-zA-Z0-9\\-\\._]|(\\\\[\\\\\\*\\?!\"#$$%&'\\(\\)\\+,/:;<=>@\\[\\]\\^`\\{\\|}~]))+(\\?*|\\*?))|[\\*\\-])){5}(:(([a-zA-Z]{2,3}(-([a-zA-Z]{2}|[0-9]{3}))?)|[\\*\\-]))(:(((\\?*|\\*?)([a-zA-Z0-9\\-\\._]|(\\\\[\\\\\\*\\?!\"#$$%&'\\(\\)\\+,/:;<=>@\\[\\]\\^`\\{\\|}~]))+(\\?*|\\*?))|[\\*\\-])){4})|([c][pP][eE]:/[AHOaho]?(:[A-Za-z0-9\\._\\-~%]*){0,6})", message = "The CPE must conform to the CPE v2.2 or v2.3 specification defined by NIST")
@@ -265,6 +266,7 @@ public enum FetchGroup {
     private String purlCoordinates; // Field should contain only type, namespace, name, and version. Everything up to the qualifiers
 
     @Persistent
+    @Column(name = "SWIDTAGID")
     @Index(name = "COMPONENT_SWID_TAGID_IDX")
     @Size(max = 255)
     @Pattern(regexp = RegexSequence.Definition.PRINTABLE_CHARS, message = "The SWID tagId may only contain printable characters")

src/main/java/org/dependencytrack/model/Project.java

@@ -203,6 +203,7 @@ public enum FetchGroup {
 
     @Persistent
     @Index(name = "PROJECT_CPE_IDX")
+    @Column(name = "CPE")
     @Size(max = 255)
     @JsonDeserialize(using = TrimmedStringDeserializer.class)
     //Patterns obtained from https://csrc.nist.gov/schema/cpe/2.3/cpe-naming_2.3.xsd
@@ -211,13 +212,15 @@ public enum FetchGroup {
 
     @Persistent
     @Index(name = "PROJECT_PURL_IDX")
+    @Column(name = "PURL")
     @Size(max = 255)
     @com.github.packageurl.validator.PackageURL
     @JsonDeserialize(using = TrimmedStringDeserializer.class)
     private String purl;
 
     @Persistent
     @Index(name = "PROJECT_SWID_TAGID_IDX")
+    @Column(name = "SWIDTAGID")
     @Size(max = 255)
     @JsonDeserialize(using = TrimmedStringDeserializer.class)
     @Pattern(regexp = RegexSequence.Definition.PRINTABLE_CHARS, message = "The SWID tagId may only contain printable characters")

src/main/java/org/dependencytrack/persistence/ProjectQueryManager.java

@@ -1265,6 +1265,7 @@ private List<ProjectVersion> getProjectVersions(Project project) {
         query.setFilter("name == :name");
         query.setParameters(project.getName());
         query.setResult("uuid, version");
+        query.setOrdering("id asc"); // Ensure consistent ordering
         return query.executeResultList(ProjectVersion.class);
     }
 }

src/main/java/org/dependencytrack/persistence/QueryManager.java

@@ -1521,9 +1521,9 @@ public void recursivelyDeleteTeam(Team team) {
         pm.currentTransaction().begin();
         pm.deletePersistentAll(team.getApiKeys());
         String aclDeleteQuery = """
-                    DELETE FROM PROJECT_ACCESS_TEAMS WHERE \"PROJECT_ACCESS_TEAMS\".\"TEAM_ID\" = ?
+                DELETE FROM "PROJECT_ACCESS_TEAMS" WHERE "TEAM_ID" = ?
                 """;
-        final Query query = pm.newQuery(JDOQuery.SQL_QUERY_LANGUAGE, aclDeleteQuery);
+        final Query<?> query = pm.newQuery(JDOQuery.SQL_QUERY_LANGUAGE, aclDeleteQuery);
         query.executeWithArray(team.getId());
         pm.deletePersistent(team);
         pm.currentTransaction().commit();

src/test/java/org/dependencytrack/PersistenceCapableTest.java

@@ -21,25 +21,48 @@
 import alpine.Config;
 import alpine.server.persistence.PersistenceManagerFactory;
 import org.apache.kafka.clients.producer.MockProducer;
+import org.datanucleus.PropertyNames;
+import org.datanucleus.api.jdo.JDOPersistenceManagerFactory;
 import org.dependencytrack.event.kafka.KafkaProducerInitializer;
 import org.dependencytrack.persistence.QueryManager;
 import org.junit.After;
+import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.contrib.java.lang.system.EnvironmentVariables;
+import org.testcontainers.containers.PostgreSQLContainer;
+
+import javax.jdo.JDOHelper;
+import java.sql.Connection;
+import java.sql.Statement;
+import java.util.Properties;
 
 public abstract class PersistenceCapableTest {
 
-    protected QueryManager qm;
+    @Rule
+    public EnvironmentVariables environmentVariables = new EnvironmentVariables();
+
+    protected static PostgresTestContainer postgresContainer;
     protected MockProducer<byte[], byte[]> kafkaMockProducer;
+    protected QueryManager qm;
 
     @BeforeClass
     public static void init() {
         Config.enableUnitTests();
+
+        postgresContainer = new PostgresTestContainer();
+        postgresContainer.start();
     }
 
     @Before
     public void before() throws Exception {
-        this.qm = new QueryManager();
+        truncateTables(postgresContainer);
+        configurePmf(postgresContainer);
+
+        qm = new QueryManager();
+
+        environmentVariables.set("TASK_PORTFOLIO_REPOMETAANALYSIS_LOCKATLEASTFORINMILLIS", "2000");
         this.kafkaMockProducer = (MockProducer<byte[], byte[]>) KafkaProducerInitializer.getProducer();
     }
 
@@ -50,11 +73,54 @@ public void after() {
         // code base can leave such a broken state behind if they run into unexpected
         // errors. See: https://github.com/DependencyTrack/dependency-track/issues/2677
         if (!qm.getPersistenceManager().isClosed()
-                && qm.getPersistenceManager().currentTransaction().isActive()) {
+            && qm.getPersistenceManager().currentTransaction().isActive()) {
             qm.getPersistenceManager().currentTransaction().rollback();
         }
+
         PersistenceManagerFactory.tearDown();
         KafkaProducerInitializer.tearDown();
     }
 
+    @AfterClass
+    public static void tearDownClass() {
+        if (postgresContainer != null) {
+            postgresContainer.stopWhenNotReusing();
+        }
+    }
+
+    protected static void configurePmf(final PostgreSQLContainer<?> postgresContainer) {
+        final var dnProps = new Properties();
+        dnProps.put(PropertyNames.PROPERTY_PERSISTENCE_UNIT_NAME, "Alpine");
+        dnProps.put(PropertyNames.PROPERTY_SCHEMA_AUTOCREATE_DATABASE, "false");
+        dnProps.put(PropertyNames.PROPERTY_SCHEMA_AUTOCREATE_TABLES, "false");
+        dnProps.put(PropertyNames.PROPERTY_SCHEMA_AUTOCREATE_COLUMNS, "false");
+        dnProps.put(PropertyNames.PROPERTY_SCHEMA_AUTOCREATE_CONSTRAINTS, "false");
+        dnProps.put("datanucleus.schema.generatedatabase.mode", "none");
+        dnProps.put("datanucleus.query.jdoql.allowall", "true");
+        dnProps.put(PropertyNames.PROPERTY_CONNECTION_URL, postgresContainer.getJdbcUrl());
+        dnProps.put(PropertyNames.PROPERTY_CONNECTION_DRIVER_NAME, postgresContainer.getDriverClassName());
+        dnProps.put(PropertyNames.PROPERTY_CONNECTION_USER_NAME, postgresContainer.getUsername());
+        dnProps.put(PropertyNames.PROPERTY_CONNECTION_PASSWORD, postgresContainer.getPassword());
+
+        final var pmf = (JDOPersistenceManagerFactory) JDOHelper.getPersistenceManagerFactory(dnProps, "Alpine");
+        PersistenceManagerFactory.setJdoPersistenceManagerFactory(pmf);
+    }
+
+    protected static void truncateTables(final PostgreSQLContainer<?> postgresContainer) throws Exception {
+        // Truncate all tables to ensure each test starts from a clean slate.
+        // https://stackoverflow.com/a/63227261
+        try (final Connection connection = postgresContainer.createConnection("");
+             final Statement statement = connection.createStatement()) {
+            statement.execute("""
+                    DO $$ DECLARE
+                        r RECORD;
+                    BEGIN
+                        FOR r IN (SELECT tablename FROM pg_tables WHERE schemaname = CURRENT_SCHEMA()) LOOP
+                            EXECUTE 'TRUNCATE TABLE ' || QUOTE_IDENT(r.tablename) || ' CASCADE';
+                        END LOOP;
+                    END $$;
+                    """);
+        }
+    }
+
 }

src/test/java/org/dependencytrack/PostgresTestContainer.java

@@ -0,0 +1,53 @@
+package org.dependencytrack;
+
+import com.github.dockerjava.api.command.InspectContainerResponse;
+import org.dependencytrack.persistence.migration.MigrationInitializer;
+import org.postgresql.ds.PGSimpleDataSource;
+import org.testcontainers.containers.PostgreSQLContainer;
+import org.testcontainers.utility.DockerImageName;
+import org.testcontainers.utility.TestcontainersConfiguration;
+
+public class PostgresTestContainer extends PostgreSQLContainer<PostgresTestContainer> {
+
+    @SuppressWarnings("resource")
+    public PostgresTestContainer() {
+        super(DockerImageName.parse("postgres:11-alpine"));
+        withUsername("dtrack");
+        withPassword("dtrack");
+        withDatabaseName("dtrack");
+        withLabel("owner", "hyades-apiserver");
+
+        // NB: Container reuse won't be active unless either:
+        //  - The environment variable TESTCONTAINERS_REUSE_ENABLE=true is set
+        //  - testcontainers.reuse.enable=false is set in ~/.testcontainers.properties
+        withReuse(true);
+    }
+
+    @Override
+    protected void containerIsStarted(final InspectContainerResponse containerInfo, final boolean reused) {
+        super.containerIsStarted(containerInfo, reused);
+
+        if (reused) {
+            logger().debug("Reusing container; Migration not necessary");
+            return;
+        }
+
+        final var dataSource = new PGSimpleDataSource();
+        dataSource.setUrl(getJdbcUrl());
+        dataSource.setUser(getUsername());
+        dataSource.setPassword(getPassword());
+
+        try {
+            MigrationInitializer.runMigration(dataSource, /* silent */ true);
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    public void stopWhenNotReusing() {
+        if (!TestcontainersConfiguration.getInstance().environmentSupportsReuse() || !isShouldBeReused()) {
+            stop();
+        }
+    }
+
+}