Skip to content

Commit

Permalink
Give SCDF ability to compile and run tests to completion
Browse files Browse the repository at this point in the history
* Disable tests that need more investigation
* Refresh the TODO and Disable messages to reflect an accurate message of what is required
* These changes do not include those for skipper
* To compile to completion comment out spring-cloud-skipper module then execute mvn clean install

Update code based on review
  • Loading branch information
cppwfs committed Mar 18, 2024
1 parent b81fa8e commit 7a6567b
Show file tree
Hide file tree
Showing 27 changed files with 108 additions and 133 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Timestamp;
import java.sql.Types;
import java.util.ArrayList;
import java.util.Collection;
Expand Down Expand Up @@ -484,11 +485,12 @@ public TaskExecution mapRow(ResultSet rs, int rowNum) throws SQLException {
if (rs.wasNull()) {
parentExecutionId = null;
}
Timestamp endTimestamp = rs.getTimestamp("END_TIME");
return new TaskExecution(id,
getNullableExitCode(rs),
rs.getString("TASK_NAME"),
rs.getTimestamp("START_TIME").toLocalDateTime(),
rs.getTimestamp("END_TIME").toLocalDateTime(),
(endTimestamp != null) ? endTimestamp.toLocalDateTime() : null,
rs.getString("EXIT_MESSAGE"),
getTaskArguments(id),
rs.getString("ERROR_MESSAGE"),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
import org.springframework.boot.cloud.CloudPlatform;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.cloud.dataflow.aggregate.task.DataflowTaskExecutionQueryDao;
import org.springframework.cloud.dataflow.aggregate.task.TaskDefinitionReader;
import org.springframework.cloud.dataflow.aggregate.task.TaskDeploymentReader;
import org.springframework.cloud.dataflow.registry.service.AppRegistryService;
Expand Down Expand Up @@ -77,6 +78,11 @@ public TaskDeploymentReader taskDeploymentReader() {
return mock(TaskDeploymentReader.class);
}

@Bean
DataflowTaskExecutionQueryDao dataflowTaskExecutionQueryDao() {
return mock(DataflowTaskExecutionQueryDao.class);
}

@Configuration
@ConditionalOnCloudPlatform(CloudPlatform.CLOUD_FOUNDRY)
public static class CloudFoundryMockConfig {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@
import java.time.LocalDateTime;

import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.ObjectCodec;
import com.fasterxml.jackson.databind.DeserializationContext;
import com.fasterxml.jackson.databind.JsonDeserializer;
Expand Down Expand Up @@ -51,7 +50,7 @@ public JobParameter deserialize(JsonParser jsonParser, DeserializationContext de
String type = node.get("type").asText();

JobParameter jobParameter;
//TODO: Boot3x followup
//TODO: Boot3x followup Verify that Job Parameters setup properly for Batch 5
if (!type.isEmpty() && !type.equalsIgnoreCase("STRING")) {
if ("DATE".equalsIgnoreCase(type)) {
jobParameter = new JobParameter(LocalDateTime.parse(value), LocalDateTime.class, identifying);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@

package org.springframework.cloud.dataflow.server.batch;

//TODO: Boot3x followup
public class JobRestartRuntimeException extends RuntimeException {

public JobRestartRuntimeException(Long jobExecutionId, Exception cause) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@

package org.springframework.cloud.dataflow.server.batch;

//TODO: Boot3x followup
public class JobStartRuntimeException extends RuntimeException {

public JobStartRuntimeException(String jobName, Exception cause) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@

package org.springframework.cloud.dataflow.server.batch;

//TODO: Boot3x followup
public class JobStopException extends RuntimeException {

public JobStopException(Long jobExecutionId, Exception cause) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,10 @@
import org.apache.hc.client5.http.impl.io.BasicHttpClientConnectionManager;
import org.apache.hc.client5.http.socket.ConnectionSocketFactory;
import org.apache.hc.client5.http.socket.PlainConnectionSocketFactory;
import org.apache.hc.client5.http.ssl.NoopHostnameVerifier;
import org.apache.hc.client5.http.ssl.SSLConnectionSocketFactory;
import org.apache.hc.core5.http.config.Lookup;
import org.apache.hc.core5.http.config.RegistryBuilder;
import org.apache.http.conn.ssl.NoopHostnameVerifier;
import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
import org.apache.hc.client5.http.impl.classic.HttpClients;
import org.slf4j.Logger;
Expand All @@ -47,6 +47,7 @@
import org.springframework.cloud.dataflow.rest.resource.about.RuntimeEnvironmentDetails;
import org.springframework.cloud.dataflow.rest.resource.about.SecurityInfo;
import org.springframework.cloud.dataflow.rest.resource.about.VersionInfo;
import org.springframework.cloud.dataflow.rest.util.HttpUtils;
import org.springframework.cloud.dataflow.server.config.DataflowMetricsProperties;
import org.springframework.cloud.dataflow.server.config.VersionInfoProperties;
import org.springframework.cloud.dataflow.server.config.features.FeaturesProperties;
Expand Down Expand Up @@ -288,7 +289,10 @@ private String getChecksum(String defaultValue, String url,
String version) {
String result = defaultValue;
if (result == null && StringUtils.hasText(url)) {
ConnectionSocketFactory sslsf = new SSLConnectionSocketFactory(HttpUtils.buildCertificateIgnoringSslContext(), NoopHostnameVerifier.INSTANCE);

Lookup<ConnectionSocketFactory> connSocketFactoryLookup = RegistryBuilder.<ConnectionSocketFactory> create()
.register("https", sslsf)
.register("http", new PlainConnectionSocketFactory())
.build();
CloseableHttpClient httpClient = HttpClients.custom()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -233,8 +233,12 @@ public void cleanupExecutions(Set<TaskExecutionControllerDeleteAction> actionsAs
.stream()
.map(TaskExecution::getExecutionId)
.collect(Collectors.toCollection(TreeSet::new));
this.performDeleteTaskExecutions(childIds);
this.performDeleteTaskExecutions(parentIds);
if(childIds.size() > 0) {
this.performDeleteTaskExecutions(childIds);
}
if(parentIds.size() > 0) {
this.performDeleteTaskExecutions(parentIds);
}
}
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -270,7 +270,7 @@ public void restartJobExecution(long jobExecutionId) throws NoSuchJobExecutionEx

}

//TODO: Boot3x followup Remove boot2 check in this method once boot2 suuport code has been removed.
//TODO: Boot3x followup Verify usage job params work with Batch 5.x
/**
* Apply identifying job parameters to arguments. There are cases (incrementers)
* that add parameters to a job and thus must be added for each restart so that the
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@
package org.springframework.cloud.dataflow.server.controller;

import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Nested;
import org.junit.jupiter.api.Test;
import org.mockito.Mockito;
Expand Down Expand Up @@ -82,8 +81,6 @@ public void setupMocks() {
.defaultRequest(get("/").accept(MediaType.APPLICATION_JSON)).build();
}

//TODO: Boot3x followup
@Disabled("Need to investigate why we can't get the RESTTemplate to resolve a https")
@Test
public void testListApplications() throws Exception {
ResultActions result = mockMvc.perform(get("/about").accept(MediaType.APPLICATION_JSON)).andDo(print()).andExpect(status().isOk());
Expand Down Expand Up @@ -394,8 +391,6 @@ public void setupMocks() {
.defaultRequest(get("/").accept(MediaType.APPLICATION_JSON)).build();
}

//TODO: Boot3x followup
@Disabled("Need to investigate why we can't get the RESTTemplate to resolve a https")
@Test
public void testAbout() throws Exception {
ResultActions result = mockMvc.perform(get("/about").accept(MediaType.APPLICATION_JSON)).andDo(print()).andExpect(status().isOk());
Expand All @@ -422,8 +417,6 @@ public void testAbout() throws Exception {
.andExpect(jsonPath("$.monitoringDashboardInfo.refreshInterval", is(30)));
}

//TODO: Boot3x followup
@Disabled("Need to investigate why we can't get the RESTTemplate to resolve a https")
@Test
public void testAboutWithMissingSkipper() throws Exception {
reset(this.skipperClient);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -146,7 +146,7 @@ public void testStopStartedJobExecution() throws Exception {
}

//TODO: Boot3x followup
@Disabled("We need to investigate why SimpleJobService uses JSR-352 for the getJobNames")
@Disabled("TODO: Boot3x followup We need to investigate why SimpleJobService uses JSR-352 for the getJobNames")
@Test
public void testStopStartedJobExecutionTwice() throws Exception {
mockMvc.perform(put("/jobs/executions/6").accept(MediaType.APPLICATION_JSON).param("stop", "true"))
Expand Down Expand Up @@ -215,7 +215,7 @@ public void testGetAllExecutions() throws Exception {
}

//TODO: Boot3x followup
@Disabled("Until we implement the paging capabilities this tests is disabled.")
@Disabled("TODO: Boot3x followup Until we implement the paging capabilities this tests is disabled.")
@Test
public void testGetAllExecutionsPageOffsetLargerThanIntMaxValue() throws Exception {
verify5XXErrorIsThrownForPageOffsetError(get("/jobs/executions"));
Expand All @@ -234,7 +234,7 @@ public void testGetExecutionsByName() throws Exception {
}

//TODO: Boot3x followup
@Disabled("Until we implement the paging capabilities this tests is disabled.")
@Disabled("TODO: Boot3x followup Until we implement the paging capabilities this tests is disabled.")
@Test
public void testGetExecutionsByNamePageOffsetLargerThanIntMaxValue() throws Exception {
verify5XXErrorIsThrownForPageOffsetError(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -177,7 +177,7 @@ public void testGetMultipleStepExecutions() throws Exception {
}

//TODO: Boot3x followup
@Disabled("Need to create DataflowSqlPagingQueryProvider so that dataflow can call generateJumpToItemQuery")
@Disabled("TODO: Boot3x followup Need to create DataflowSqlPagingQueryProvider so that dataflow can call generateJumpToItemQuery")
public void testSingleGetStepExecutionProgress() throws Exception {
mockMvc.perform(get("/jobs/executions/1/steps/1/progress").accept(MediaType.APPLICATION_JSON))
.andDo(print())
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@

import org.awaitility.Awaitility;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;

import org.springframework.beans.factory.annotation.Autowired;
Expand Down Expand Up @@ -116,8 +115,6 @@ public void setupMockMVC() {
.defaultRequest(get("/").accept(MediaType.APPLICATION_JSON)).build();
}

//TODO: Boot3x followup
@Disabled("TODO: Boot3 followup")
@Test
void cleanupAll() throws Exception {
String taskExecutionId = "asyncCleanupAllTaskExecId";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,6 @@
import com.fasterxml.jackson.datatype.jdk8.Jdk8Module;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;

import org.springframework.batch.core.JobExecution;
Expand Down Expand Up @@ -106,8 +105,6 @@
* @author Corneil du Plessis
*/

//TODO: Boot3x followup
@Disabled("TODO: Boot3 followup after boot3/boot2 task changes are complete")
@SpringBootTest(
classes = { JobDependencies.class, TaskExecutionAutoConfiguration.class, DataflowAsyncAutoConfiguration.class,
PropertyPlaceholderAutoConfiguration.class, BatchProperties.class})
Expand Down Expand Up @@ -378,7 +375,7 @@ void boot3Execution() throws Exception {
mapper.registerModule(new Jackson2DataflowModule());
LaunchResponseResource resource = mapper.readValue(response, LaunchResponseResource.class);
resultActions = mockMvc.perform(
get("/tasks/executions" + resource.getExecutionId())
get("/tasks/executions/" + resource.getExecutionId())
.accept(MediaType.APPLICATION_JSON))
.andDo(print())
.andExpect(status().isOk())
Expand All @@ -393,36 +390,10 @@ void boot3Execution() throws Exception {
assertThat(json.findValue("deploymentProperties")).isNotNull();
JsonNode deploymentProperties = json.findValue("deploymentProperties");
System.out.println("deploymentProperties=" + deploymentProperties.toPrettyString());
assertThat(deploymentProperties.hasNonNull("app.timestamp3.spring.cloud.task.tablePrefix")).isTrue();
assertThat(deploymentProperties.get("app.timestamp3.spring.cloud.task.tablePrefix").asText()).isEqualTo("BOOT3_TASK_");
}

@Test
void invalidBoot3Execution() throws Exception {
if (appRegistryService.getDefaultApp("timestamp3", ApplicationType.task) == null) {
appRegistryService.save("timestamp3",
ApplicationType.task,
"3.0.0",
new URI("file:src/test/resources/apps/foo-task"),
null,
AppBootSchemaVersion.BOOT3);
}
taskDefinitionRepository.save(new TaskDefinition("timestamp3", "timestamp3"));
when(taskLauncher.launch(any())).thenReturn("abc");

ResultActions resultActions = mockMvc.perform(
post("/tasks/executions")
.queryParam("name", "timestamp3")
.accept(MediaType.APPLICATION_JSON)
).andDo(print())
.andExpect(status().isBadRequest());

String response = resultActions.andReturn().getResponse().getContentAsString();
assertThat(response).contains("cannot be launched for");
}

@Test
void boot2Execution() throws Exception {
void bootExecution() throws Exception {
if (appRegistryService.getDefaultApp("timestamp2", ApplicationType.task) == null) {
appRegistryService.save("timestamp2",
ApplicationType.task,
Expand Down Expand Up @@ -450,7 +421,7 @@ void boot2Execution() throws Exception {
mapper.registerModule(new Jackson2DataflowModule());
LaunchResponseResource resource = mapper.readValue(response, LaunchResponseResource.class);
resultActions = mockMvc.perform(
get("/tasks/executions" + resource.getExecutionId())
get("/tasks/executions/" + resource.getExecutionId())
.accept(MediaType.APPLICATION_JSON))
.andDo(print())
.andExpect(status().isOk())
Expand All @@ -465,15 +436,13 @@ void boot2Execution() throws Exception {
assertThat(json.findValue("deploymentProperties")).isNotNull();
JsonNode deploymentProperties = json.findValue("deploymentProperties");
System.out.println("deploymentProperties=" + deploymentProperties.toPrettyString());
assertThat(deploymentProperties.hasNonNull("app.timestamp2.spring.cloud.task.tablePrefix")).isTrue();
assertThat(deploymentProperties.get("app.timestamp2.spring.cloud.task.tablePrefix").asText()).isEqualTo("TASK_");

}

@Test
void getExecutionsByName() throws Exception {
verifyTaskArgs(SAMPLE_CLEANSED_ARGUMENT_LIST, "$._embedded.taskExecutionResourceList[0].",
mockMvc.perform(get("/tasks/executions/").param("name", TASK_NAME_ORIG).accept(MediaType.APPLICATION_JSON))
mockMvc.perform(get("/tasks/executions").param("name", TASK_NAME_ORIG).accept(MediaType.APPLICATION_JSON))
.andDo(print())
.andExpect(status().isOk()))
.andExpect(jsonPath("$._embedded.taskExecutionResourceList[0].taskName", is(TASK_NAME_ORIG)))
Expand All @@ -485,7 +454,7 @@ void getExecutionsByName() throws Exception {

@Test
void getExecutionsByNameNotFound() throws Exception {
mockMvc.perform(get("/tasks/executions/").param("name", "BAZ").accept(MediaType.APPLICATION_JSON))
mockMvc.perform(get("/tasks/executions").param("name", "BAZ").accept(MediaType.APPLICATION_JSON))
.andDo(print())
.andExpect(status().is4xxClientError()).andReturn().getResponse().getContentAsString()
.contains("NoSuchTaskException");
Expand Down Expand Up @@ -537,7 +506,7 @@ void cleanupByIdNotFound() throws Exception {
@Test
void deleteSingleTaskExecutionById() throws Exception {
verifyTaskArgs(SAMPLE_CLEANSED_ARGUMENT_LIST, "$._embedded.taskExecutionResourceList[0].",
mockMvc.perform(get("/tasks/executions/").accept(MediaType.APPLICATION_JSON))
mockMvc.perform(get("/tasks/executions").accept(MediaType.APPLICATION_JSON))
.andDo(print())
.andExpect(status().isOk()))
.andExpect(jsonPath("$._embedded.taskExecutionResourceList[*].executionId", containsInAnyOrder(4, 3, 2, 1)))
Expand All @@ -546,7 +515,7 @@ void deleteSingleTaskExecutionById() throws Exception {
.andDo(print())
.andExpect(status().isOk());
verifyTaskArgs(SAMPLE_CLEANSED_ARGUMENT_LIST, "$._embedded.taskExecutionResourceList[0].",
mockMvc.perform(get("/tasks/executions/").accept(MediaType.APPLICATION_JSON))
mockMvc.perform(get("/tasks/executions").accept(MediaType.APPLICATION_JSON))
.andDo(print())
.andExpect(status().isOk()))
.andExpect(jsonPath("$._embedded.taskExecutionResourceList[*].executionId", containsInAnyOrder(4, 3)))
Expand All @@ -561,7 +530,7 @@ void deleteSingleTaskExecutionById() throws Exception {
@Test
void deleteThreeTaskExecutionsById() throws Exception {
verifyTaskArgs(SAMPLE_CLEANSED_ARGUMENT_LIST, "$._embedded.taskExecutionResourceList[0].",
mockMvc.perform(get("/tasks/executions/").accept(MediaType.APPLICATION_JSON))
mockMvc.perform(get("/tasks/executions").accept(MediaType.APPLICATION_JSON))
.andDo(print())
.andExpect(status().isOk()))
.andExpect(jsonPath("$._embedded.taskExecutionResourceList[*].executionId", containsInAnyOrder(4, 3, 2, 1)))
Expand All @@ -570,7 +539,7 @@ void deleteThreeTaskExecutionsById() throws Exception {
.andDo(print())
.andExpect(status().isOk());
verifyTaskArgs(SAMPLE_CLEANSED_ARGUMENT_LIST, "$._embedded.taskExecutionResourceList[0].",
mockMvc.perform(get("/tasks/executions/").accept(MediaType.APPLICATION_JSON))
mockMvc.perform(get("/tasks/executions").accept(MediaType.APPLICATION_JSON))
.andDo(print())
.andExpect(status().isOk()))
.andExpect(jsonPath("$._embedded.taskExecutionResourceList[*].executionId", containsInAnyOrder(4)))
Expand All @@ -580,16 +549,13 @@ void deleteThreeTaskExecutionsById() throws Exception {
@Test
void deleteAllTaskExecutions() throws Exception {
verifyTaskArgs(SAMPLE_CLEANSED_ARGUMENT_LIST, "$._embedded.taskExecutionResourceList[0].",
mockMvc.perform(get("/tasks/executions/").accept(MediaType.APPLICATION_JSON))
.andDo(print())
mockMvc.perform(get("/tasks/executions").accept(MediaType.APPLICATION_JSON))
.andExpect(status().isOk()))
.andExpect(jsonPath("$._embedded.taskExecutionResourceList[*].executionId", containsInAnyOrder(4, 3, 2, 1)))
.andExpect(jsonPath("$._embedded.taskExecutionResourceList", hasSize(4)));
mockMvc.perform(delete("/tasks/executions").param("action", "CLEANUP,REMOVE_DATA"))
.andDo(print())
.andExpect(status().isOk());
mockMvc.perform(get("/tasks/executions/").accept(MediaType.APPLICATION_JSON))
.andDo(print())
mockMvc.perform(get("/tasks/executions").accept(MediaType.APPLICATION_JSON))
.andExpect(status().isOk())
.andExpect(jsonPath("$.page.totalElements", is(0)));
}
Expand All @@ -611,13 +577,6 @@ void sorting() throws Exception {
.andDo(print())
.andExpect(status().isOk());

mockMvc.perform(get("/tasks/executions").param("sort", "SCHEMA_TARGET").accept(MediaType.APPLICATION_JSON))
.andDo(print())
.andExpect(status().isOk());
mockMvc.perform(get("/tasks/executions").param("sort", "schema_target").accept(MediaType.APPLICATION_JSON))
.andDo(print())
.andExpect(status().isOk());

mockMvc.perform(get("/tasks/executions").param("sort", "WRONG_FIELD").accept(MediaType.APPLICATION_JSON))
.andDo(print())
.andExpect(status().is5xxServerError())
Expand Down
Loading

0 comments on commit 7a6567b

Please sign in to comment.