Merge pull request #39 from DiSSCo/feature/update-cron

Update files
DiSSCo · Apr 11, 2024 · 34a6597 · 34a6597
2 parents fdeb777 + 1f0917b
commit 34a6597
Show file tree

Hide file tree

Showing 10 changed files with 74 additions and 137 deletions.
diff --git a/kubernetes-roles.yaml b/kubernetes-roles.yaml
diff --git a/kubernetes-route.yaml b/kubernetes-route.yaml
diff --git a/kubernetes.yaml b/kubernetes.yaml
diff --git a/src/main/java/eu/dissco/orchestration/backend/properties/TranslatorJobProperties.java b/src/main/java/eu/dissco/orchestration/backend/properties/TranslatorJobProperties.java
@@ -16,12 +16,12 @@ public class TranslatorJobProperties {
   private String image = "public.ecr.aws/dissco/dissco-core-translator:latest";
 
   @NotBlank
-  private String kafkaHost = "kafka.kafka.svc.cluster.local:9092";
+  private String kafkaHost = "kafka-cluster-kafka-bootstrap.kafka.svc.cluster.local:9092";
 
   @NotBlank
-  private String kafkaTopic = "digital-specimen";
+  private String kafkaTopic = "col";
 
   @NotBlank
-  private String namespace = "default";
+  private String namespace = "translator-services";
 
 }
diff --git a/src/main/java/eu/dissco/orchestration/backend/service/SourceSystemService.java b/src/main/java/eu/dissco/orchestration/backend/service/SourceSystemService.java
@@ -36,6 +36,7 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Random;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.RandomStringUtils;
@@ -59,14 +60,15 @@ public class SourceSystemService {
   private final TranslatorJobProperties jobProperties;
   private final Configuration configuration;
   private final BatchV1Api batchV1Api;
+  private final Random random;
 
   private static String getSuffix(String sourceSystemId) {
     return sourceSystemId.substring(sourceSystemId.indexOf('/') + 1).toLowerCase();
   }
 
   private static String generateJobName(SourceSystemRecord sourceSystem, boolean isCron) {
     var name =
-        sourceSystem.sourceSystem().translatorType().getName().toLowerCase() + "-" +
+        sourceSystem.sourceSystem().translatorType().getLiteral().toLowerCase() + "-" +
             getSuffix(sourceSystem.id()) + "-translator-service";
     if (!isCron) {
       name = name + "-" + RandomStringUtils.randomAlphabetic(6).toLowerCase();
@@ -336,6 +338,7 @@ private Map<String, Object> getTemplateProperties(SourceSystemRecord sourceSyste
     map.put("image", jobProperties.getImage());
     map.put("sourceSystemId", sourceSystem.id());
     map.put("jobName", jobName);
+    map.put("namespace", jobProperties.getNamespace());
     map.put("containerName", jobName);
     map.put("kafkaHost", jobProperties.getKafkaHost());
     map.put("kafkaTopic", jobProperties.getKafkaTopic());
@@ -346,8 +349,8 @@ private Map<String, Object> getTemplateProperties(SourceSystemRecord sourceSyste
   }
 
   private String generateCron() {
-    String day = RandomStringUtils.randomNumeric(0, 6);
-    String hour = RandomStringUtils.randomNumeric(0, 23);
+    String day = String.valueOf(random.nextInt(7));
+    String hour = String.valueOf(random.nextInt(23));
     return "0 " + hour + " * * " + day;
   }
 

diff --git a/src/main/resources/templates/biocase-cron-job.ftl b/src/main/resources/templates/biocase-cron-job.ftl
@@ -2,13 +2,15 @@ apiVersion: batch/v1
 kind: CronJob
 metadata:
   name: ${jobName}
+  namespace: ${namespace}
 spec:
   schedule: ${cron}
   jobTemplate:
     spec:
       template:
         spec:
           restartPolicy: Never
+          serviceAccountName: translator-secret-manager
           containers:
           - name: ${containerName}
             image: ${image}
@@ -42,8 +44,19 @@ spec:
               value: https://doi.org/21.T11148/bbad8c4e101e8af01115
             - name: fdo.digital-specimen-type
               value: https://doi.org/21.T11148/894b1e6cad57e921764e
-            - name: JAVA_OPTS
-              value: -server -XX:+useContainerSupport -XX:MaxRAMPercentage=75 --illegal-access=deny
+            - name: JAVA_TOOL_OPTIONS
+              value: -XX:MaxRAMPercentage=85
             securityContext:
               runAsNonRoot: true
               allowPrivilegeEscalation: false
+            volumeMounts:
+              - name: db-secrets
+                mountPath: "/mnt/secrets-store/db-secrets"
+                readOnly: true
+          volumes:
+            - name: db-secrets
+              csi:
+                driver: secrets-store.csi.k8s.io
+                readOnly: true
+                volumeAttributes:
+                  secretProviderClass: "db-secrets"
diff --git a/src/main/resources/templates/biocase-translator-job.ftl b/src/main/resources/templates/biocase-translator-job.ftl
@@ -2,11 +2,13 @@ apiVersion: batch/v1
 kind: Job
 metadata:
   name: ${jobName}
+  namespace: ${namespace}
 spec:
   backoffLimit: 2
   template:
     spec:
       restartPolicy: Never
+      serviceAccountName: translator-secret-manager
       containers:
         - name: ${containerName}
           image: ${image}
@@ -40,8 +42,19 @@ spec:
               value: https://doi.org/21.T11148/bbad8c4e101e8af01115
             - name: fdo.digital-specimen-type
               value: https://doi.org/21.T11148/894b1e6cad57e921764e
-            - name: JAVA_OPTS
-              value: -server -XX:+useContainerSupport -XX:MaxRAMPercentage=75 --illegal-access=deny
+            - name: JAVA_TOOL_OPTIONS
+              value: -XX:MaxRAMPercentage=85
           securityContext:
             runAsNonRoot: true
             allowPrivilegeEscalation: false
+          volumeMounts:
+            - name: db-secrets
+              mountPath: "/mnt/secrets-store/db-secrets"
+              readOnly: true
+      volumes:
+        - name: db-secrets
+          csi:
+            driver: secrets-store.csi.k8s.io
+            readOnly: true
+            volumeAttributes:
+              secretProviderClass: "db-secrets"
diff --git a/src/main/resources/templates/dwca-cron-job.ftl b/src/main/resources/templates/dwca-cron-job.ftl
@@ -2,12 +2,14 @@ apiVersion: batch/v1
 kind: CronJob
 metadata:
   name: ${jobName}
+  namespace: ${namespace}
 spec:
   schedule: ${cron}
   jobTemplate:
     spec:
       template:
         spec:
+          serviceAccountName: translator-secret-manager
           restartPolicy: Never
           containers:
           - name: ${containerName}
@@ -46,14 +48,23 @@ spec:
               value: https://doi.org/21.T11148/bbad8c4e101e8af01115
             - name: fdo.digital-specimen-type
               value: https://doi.org/21.T11148/894b1e6cad57e921764e
-            - name: JAVA_OPTS
-              value: -server -XX:+useContainerSupport -XX:MaxRAMPercentage=75 --illegal-access=deny
+            - name: JAVA_TOOL_OPTIONS
+              value: -XX:MaxRAMPercentage=85
             securityContext:
               runAsNonRoot: true
               allowPrivilegeEscalation: false
             volumeMounts:
-            - mountPath: /temp
-              name: temp-volume
+              - mountPath: /temp
+                name: temp-volume
+              - name: db-secrets
+                mountPath: "/mnt/secrets-store/db-secrets"
+                readOnly: true
           volumes:
-          - name: temp-volume
-            emptyDir: { }
+            - name: temp-volume
+              emptyDir: { }
+            - name: db-secrets
+              csi:
+                driver: secrets-store.csi.k8s.io
+                readOnly: true
+                volumeAttributes:
+                  secretProviderClass: "db-secrets"
diff --git a/src/main/resources/templates/dwca-translator-job.ftl b/src/main/resources/templates/dwca-translator-job.ftl
@@ -2,11 +2,13 @@ apiVersion: batch/v1
 kind: Job
 metadata:
   name: ${jobName}
+  namespace: ${namespace}
 spec:
   backoffLimit: 2
   template:
     spec:
       restartPolicy: Never
+      serviceAccountName: translator-secret-manager
       containers:
         - name: ${containerName}
           image: ${image}
@@ -44,14 +46,23 @@ spec:
               value: https://doi.org/21.T11148/bbad8c4e101e8af01115
             - name: fdo.digital-specimen-type
               value: https://doi.org/21.T11148/894b1e6cad57e921764e
-            - name: JAVA_OPTS
-              value: -server -XX:+useContainerSupport -XX:MaxRAMPercentage=75 --illegal-access=deny
+            - name: JAVA_TOOL_OPTIONS
+              value: -XX:MaxRAMPercentage=85
           securityContext:
             runAsNonRoot: true
             allowPrivilegeEscalation: false
           volumeMounts:
             - mountPath: /temp
               name: temp-volume
+            - name: db-secrets
+              mountPath: "/mnt/secrets-store/db-secrets"
+              readOnly: true
       volumes:
         - name: temp-volume
-          emptyDir: { }
+          emptyDir: { }
+        - name: db-secrets
+          csi:
+            driver: secrets-store.csi.k8s.io
+            readOnly: true
+            volumeAttributes:
+              secretProviderClass: "db-secrets"
diff --git a/src/test/java/eu/dissco/orchestration/backend/service/SourceSystemServiceTest.java b/src/test/java/eu/dissco/orchestration/backend/service/SourceSystemServiceTest.java
@@ -60,6 +60,7 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.Optional;
+import java.util.Random;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
@@ -71,7 +72,7 @@
 @ExtendWith(MockitoExtension.class)
 class SourceSystemServiceTest {
 
-  private static final String NAMESPACE = "default";
+  private static final String NAMESPACE = "translator-services";
 
   private final ObjectMapper yamlMapper = new ObjectMapper(
       new YAMLFactory()).findAndRegisterModules();
@@ -92,6 +93,7 @@ class SourceSystemServiceTest {
   private MappingService mappingService;
   @Mock
   private BatchV1Api batchV1Api;
+  private Random random = new Random();
 
 
   private MockedStatic<Instant> mockedStatic;
@@ -100,7 +102,7 @@ class SourceSystemServiceTest {
   @BeforeEach
   void setup() throws IOException {
     service = new SourceSystemService(builder, handleComponent, repository, mappingService,
-        kafkaPublisherService, MAPPER, yamlMapper, jobProperties, configuration, batchV1Api);
+        kafkaPublisherService, MAPPER, yamlMapper, jobProperties, configuration, batchV1Api, random);
     initTime();
     initFreeMaker();
   }