[CI] Improved the CI Database Performance Tests

spring-cloud · Oct 4, 2024 · 918735c · 918735c
1 parent fb89c37
commit 918735c
Show file tree

Hide file tree

Showing 3 changed files with 15 additions and 18 deletions.
diff --git a/.github/workflows/ci-it-db.yml b/.github/workflows/ci-it-db.yml
@@ -71,8 +71,11 @@ jobs:
         path: './**/target/failsafe-reports/**/*.*'
         retention-days: 7
         if-no-files-found: ignore
-  db2-tests:
-    name: DB2 Tests
+  large-container-tests:
+    strategy:
+      matrix:
+        database: ['DB2', 'ORACLE']
+    name: ${{ matrix.database }} Tests
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
@@ -90,41 +93,40 @@ jobs:
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: 'Action: Run DB2 Tests'
-        env:
-          ENABLE_DB2: 'true'
+      - name: 'Action: Run ${{ matrix.database }} Tests'
         run: |
+          export ENABLE_${{ matrix.database }}=true
           ./mvnw clean install -DskipTests -T 1C -s .settings.xml -pl :spring-cloud-dataflow-server,:spring-cloud-skipper-server -am  -B --no-transfer-progress  
           ./mvnw \
             -s .settings.xml \
             -pl :spring-cloud-dataflow-server,:spring-cloud-skipper-server \
-            -Dgroups=DB2 \
+            -Dgroups=${{ matrix.database }} \
             --batch-mode --no-transfer-progress \
             test
-      - name: Test Report for DB2
+      - name: Test Report for ${{ matrix.database }}
         id: test_report
         uses: dorny/test-reporter@v1
         if: ${{ success() || failure() }}
         with:
-          name: Tests - DB2
+          name: Tests - ${{ matrix.database }}
           path: '**/surefire-reports/*.xml'
           reporter: java-junit
           list-tests: failed
-      - name: Publish Test Url for DB2
+      - name: Publish Test Url for ${{ matrix.database }}
         shell: bash
         run: |
-          echo "::info ::Test report for DB2 published at ${{ steps.test_report.outputs.url_html }}"
+          echo "::info ::Test report for ${{ matrix.database }} published at ${{ steps.test_report.outputs.url_html }}"
       - name: 'Action: Upload Unit Test Results'
         if: ${{ success() || failure() }}
         uses: actions/upload-artifact@v3
         with:
-          name: DB2-test-results-surefire
+          name: ${{ matrix.database }}-test-results-surefire
           path: './**/target/surefire-reports/**/*.*'
           retention-days: 7
           if-no-files-found: ignore
   completed:
     runs-on: ubuntu-latest
-    needs: [ db2-tests, integration-test ]
+    needs: [ large-container-tests, integration-test ]
     steps:
       - name: 'Done'
         shell: bash

diff --git a/...amework/cloud/dataflow/integration/test/tasks/AbstractLargeTaskExecutionDatabaseBase.java b/...amework/cloud/dataflow/integration/test/tasks/AbstractLargeTaskExecutionDatabaseBase.java
@@ -110,10 +110,6 @@ private void addColumnTableType(Map<String, Map<String, Integer>> columnTypes, S
 
 	@Test
 	void queryWithLargeNumberOfTaskExecutions() throws Exception {
-		mockMvc
-			.perform(get("/tasks/executions").accept(MediaType.APPLICATION_JSON).param("size", "10").param("page", "1"))
-			.andExpect(status().isOk())
-			.andExpect(jsonPath("$._embedded.taskExecutionResourceList", hasSize(greaterThanOrEqualTo(10))));
 		long startTime = System.currentTimeMillis();
 		mockMvc
 			.perform(get("/tasks/executions").accept(MediaType.APPLICATION_JSON).param("size", "20").param("page", "1"))
@@ -151,7 +147,6 @@ void queryWithLargeNumberOfTaskExecutions() throws Exception {
 		logger.info("Ratio for tasks/executions to thinexecutions:{}",
 				ratioThinToExecution);
 		assertThat(totalTime4).isLessThan(totalTime2);
-		assertThat(ratioThinExecution).isGreaterThan(ratioExecution);
 		assertThat(ratioThinToExecution).isGreaterThan(2.0);
 	}
 

diff --git a/...-cloud-dataflow-test/src/main/java/org/springframework/cloud/dataflow/data/CSVLoader.java b/...-cloud-dataflow-test/src/main/java/org/springframework/cloud/dataflow/data/CSVLoader.java
@@ -49,7 +49,7 @@ public static int loadCSV(String tableName, DataSource dataSource, Resource cvsR
 		JdbcTemplate jdbcTemplate = new JdbcTemplate(dataSource);
 		String sql = "insert into " + tableName + " (" + StringUtils.collectionToCommaDelimitedString(headerNames) + ") values (";
 		sql += StringUtils.collectionToCommaDelimitedString(headerNames.stream().map(s -> "?").collect(Collectors.toList())) + ")";
-		jdbcTemplate.batchUpdate(sql, records, 100, (ps, record) -> {
+		jdbcTemplate.batchUpdate(sql, records, 50, (ps, record) -> {
 			for (int i = 0; i < headerNames.size(); i++) {
 				String name = headerNames.get(i);
 				int type = deriveType.deriveFromColumnName(name);