Feature/add ingestion cron

.github/workflows/.trivyignore

@@ -1,4 +1,4 @@
-# Date: November 14, 2023
-# Issue: openssl: Incorrect cipher key and IV length processing https://avd.aquasec.com/nvd/cve-2023-5363
-# Solution: Docker image needs an update
-CVE-2023-5363
+# Date: Feb 27, 2024
+# Notes: Issue with libexpat, parsing large tokens can trigger a denial of service
+# Needs to be fixed in Docker Image.
+CVE-2023-52425

pom.xml

@@ -6,7 +6,7 @@
   <parent>
     <groupId>org.springframework.boot</groupId>
     <artifactId>spring-boot-starter-parent</artifactId>
-    <version>3.2.0</version>
+    <version>3.2.3</version>
     <relativePath/> <!-- lookup parent from repository -->
   </parent>
   <groupId>eu.dissco.core</groupId>
@@ -16,17 +16,17 @@
   <description>Backend for orchestration service</description>
   <properties>
     <java.version>17</java.version>
-    <apachecommons.version>4.3</apachecommons.version>
-    <k8s-client-java.version>19.0.0</k8s-client-java.version>
-    <mockito-inline.version>5.2.0</mockito-inline.version>
+    <apachecommons.version>4.4</apachecommons.version>
+    <apachecommons-compress.version>1.26.0</apachecommons-compress.version>
+    <k8s-client-java.version>20.0.0</k8s-client-java.version>
     <json-patch.version>1.13</json-patch.version>
+    <testcontainers.version>1.19.6</testcontainers.version>
+    <mockito-inline.version>5.2.0</mockito-inline.version>
     <sonar.organization>dissco</sonar.organization>
-    <logback.version>1.4.12</logback.version>
     <sonar.host.url>https://sonarcloud.io</sonar.host.url>
     <sonar.coverage.jacoco.xmlReportPaths>
       ../app-it/target/site/jacoco-aggregate/jacoco.xml
     </sonar.coverage.jacoco.xmlReportPaths>
-    <testcontainers.version>1.17.6</testcontainers.version>
   </properties>
 
   <dependencyManagement>
@@ -115,6 +115,12 @@
       <artifactId>json-patch</artifactId>
       <version>${json-patch.version}</version>
     </dependency>
+    <!-- Added because of CVE-2024-25710 and CVE-2024-26308 in kuberentes and testcontainer  -->
+    <dependency>
+      <groupId>org.apache.commons</groupId>
+      <artifactId>commons-compress</artifactId>
+      <version>${apachecommons-compress.version}</version>
+    </dependency>
 
     <dependency>
       <groupId>org.junit.jupiter</groupId>

src/main/java/eu/dissco/orchestration/backend/DemoApplication.java → src/main/java/eu/dissco/orchestration/backend/DiSSCoBackendApplication.java

@@ -6,10 +6,10 @@
 
 @SpringBootApplication
 @ConfigurationPropertiesScan
-public class DemoApplication {
+public class DiSSCoBackendApplication {
 
   public static void main(String[] args) {
-    SpringApplication.run(DemoApplication.class, args);
+    SpringApplication.run(DiSSCoBackendApplication.class, args);
   }
 
 }

src/main/java/eu/dissco/orchestration/backend/controller/MachineAnnotationServiceController.java

@@ -8,6 +8,7 @@
 import eu.dissco.orchestration.backend.domain.jsonapi.JsonApiRequestWrapper;
 import eu.dissco.orchestration.backend.domain.jsonapi.JsonApiWrapper;
 import eu.dissco.orchestration.backend.exception.NotFoundException;
+import eu.dissco.orchestration.backend.exception.ProcessingFailedException;
 import eu.dissco.orchestration.backend.properties.ApplicationProperties;
 import eu.dissco.orchestration.backend.service.MachineAnnotationServiceService;
 import jakarta.servlet.http.HttpServletRequest;
@@ -43,7 +44,7 @@ public class MachineAnnotationServiceController {
   public ResponseEntity<JsonApiWrapper> createMachineAnnotationService(
       Authentication authentication,
       @RequestBody JsonApiRequestWrapper requestBody, HttpServletRequest servletRequest)
-      throws JsonProcessingException, TransformerException {
+      throws JsonProcessingException, ProcessingFailedException {
     var machineAnnotationService = getMachineAnnotation(requestBody);
     var userId = authentication.getName();
     log.info("Received create request for machine annotation service: {} from user: {}",
@@ -58,7 +59,7 @@ public ResponseEntity<JsonApiWrapper> updateMachineAnnotationService(
       Authentication authentication,
       @PathVariable("prefix") String prefix, @PathVariable("suffix") String suffix,
       @RequestBody JsonApiRequestWrapper requestBody, HttpServletRequest servletRequest)
-      throws JsonProcessingException, NotFoundException {
+      throws JsonProcessingException, NotFoundException, ProcessingFailedException {
     var machineAnnotationService = getMachineAnnotation(requestBody);
     var userId = authentication.getName();
     var id = prefix + '/' + suffix;
@@ -77,7 +78,7 @@ public ResponseEntity<JsonApiWrapper> updateMachineAnnotationService(
   @DeleteMapping(value = "/{prefix}/{suffix}", produces = MediaType.APPLICATION_JSON_VALUE)
   public ResponseEntity<Void> deleteMachineAnnotationService(Authentication authentication,
       @PathVariable("prefix") String prefix, @PathVariable("suffix") String suffix)
-      throws NotFoundException {
+      throws NotFoundException, ProcessingFailedException {
     String id = prefix + "/" + suffix;
     log.info("Received delete request for machine annotation service: {} from user: {}", id,
         authentication.getName());

src/main/java/eu/dissco/orchestration/backend/controller/MappingController.java

@@ -8,6 +8,7 @@
 import eu.dissco.orchestration.backend.domain.jsonapi.JsonApiRequestWrapper;
 import eu.dissco.orchestration.backend.domain.jsonapi.JsonApiWrapper;
 import eu.dissco.orchestration.backend.exception.NotFoundException;
+import eu.dissco.orchestration.backend.exception.ProcessingFailedException;
 import eu.dissco.orchestration.backend.properties.ApplicationProperties;
 import eu.dissco.orchestration.backend.service.MappingService;
 import jakarta.servlet.http.HttpServletRequest;
@@ -45,7 +46,7 @@ public class MappingController {
   @PostMapping(consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
   public ResponseEntity<JsonApiWrapper> createMapping(Authentication authentication,
       @RequestBody JsonApiRequestWrapper requestBody, HttpServletRequest servletRequest)
-      throws JsonProcessingException {
+      throws JsonProcessingException, ProcessingFailedException {
     var mapping = getMappingFromRequest(requestBody);
     var userId = authentication.getName();
     log.info("Received create request for mapping: {} from user: {}", mapping, userId);
@@ -58,7 +59,7 @@ public ResponseEntity<JsonApiWrapper> createMapping(Authentication authenticatio
   public ResponseEntity<JsonApiWrapper> updateMapping(Authentication authentication,
       @PathVariable("prefix") String prefix, @PathVariable("suffix") String suffix,
       @RequestBody JsonApiRequestWrapper requestBody, HttpServletRequest servletRequest)
-      throws JsonProcessingException, NotFoundException {
+      throws JsonProcessingException, NotFoundException, ProcessingFailedException {
     var mapping = getMappingFromRequest(requestBody);
     var id = prefix + '/' + suffix;
     var userId = authentication.getName();

src/main/java/eu/dissco/orchestration/backend/controller/SourceSystemController.java

@@ -8,9 +8,13 @@
 import eu.dissco.orchestration.backend.domain.jsonapi.JsonApiRequestWrapper;
 import eu.dissco.orchestration.backend.domain.jsonapi.JsonApiWrapper;
 import eu.dissco.orchestration.backend.exception.NotFoundException;
+import eu.dissco.orchestration.backend.exception.ProcessingFailedException;
 import eu.dissco.orchestration.backend.properties.ApplicationProperties;
 import eu.dissco.orchestration.backend.service.SourceSystemService;
+import freemarker.template.TemplateException;
+import io.kubernetes.client.openapi.ApiException;
 import jakarta.servlet.http.HttpServletRequest;
+import java.io.IOException;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.http.HttpStatus;
@@ -41,7 +45,7 @@ public class SourceSystemController {
   @PostMapping(consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
   public ResponseEntity<JsonApiWrapper> createSourceSystem(Authentication authentication,
       @RequestBody JsonApiRequestWrapper requestBody, HttpServletRequest servletRequest)
-      throws JsonProcessingException, NotFoundException {
+      throws IOException, NotFoundException, ProcessingFailedException {
     var sourceSystem = getSourceSystemFromRequest(requestBody);
     String path = appProperties.getBaseUrl() + servletRequest.getRequestURI();
     var userId = authentication.getName();
@@ -54,7 +58,7 @@ public ResponseEntity<JsonApiWrapper> createSourceSystem(Authentication authenti
   public ResponseEntity<JsonApiWrapper> updateSourceSystem(Authentication authentication,
       @PathVariable("prefix") String prefix, @PathVariable("suffix") String suffix,
       @RequestBody JsonApiRequestWrapper requestBody, HttpServletRequest servletRequest)
-      throws JsonProcessingException, NotFoundException {
+      throws IOException, NotFoundException, ProcessingFailedException {
     var sourceSystem = getSourceSystemFromRequest(requestBody);
     var id = prefix + '/' + suffix;
     var userId = authentication.getName();
@@ -72,7 +76,7 @@ public ResponseEntity<JsonApiWrapper> updateSourceSystem(Authentication authenti
   @DeleteMapping(value = "/{prefix}/{suffix}", produces = MediaType.APPLICATION_JSON_VALUE)
   public ResponseEntity<Void> deleteSourceSystem(Authentication authentication,
       @PathVariable("prefix") String prefix, @PathVariable("suffix") String suffix)
-      throws NotFoundException {
+      throws NotFoundException, ProcessingFailedException {
     String id = prefix + "/" + suffix;
     log.info("Received delete request for mapping: {} from user: {}", id,
         authentication.getName());
@@ -91,6 +95,17 @@ public ResponseEntity<JsonApiWrapper> getSourceSystemById(@PathVariable("prefix"
     return ResponseEntity.ok(sourceSystem);
   }
 
+  @ResponseStatus(HttpStatus.OK)
+  @PostMapping(value = "/{prefix}/{suffix}/run", produces = MediaType.APPLICATION_JSON_VALUE)
+  public ResponseEntity<JsonApiWrapper> scheduleRunSourceSystemById(
+      @PathVariable("prefix") String prefix, @PathVariable("suffix") String suffix)
+      throws ProcessingFailedException {
+    var id = prefix + '/' + suffix;
+    log.info("Received a request to start a new run for Source System: {}", id);
+    service.runSourceSystemById(id);
+    return ResponseEntity.accepted().build();
+  }
+
   @GetMapping("")
   public ResponseEntity<JsonApiListWrapper> getSourceSystems(
       @RequestParam(value = "pageNumber", defaultValue = "1") int pageNum,