diff --git a/kubernetes-roles.yaml b/kubernetes-roles.yaml deleted file mode 100644 index 1b15980..0000000 --- a/kubernetes-roles.yaml +++ /dev/null @@ -1,31 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: orchestration-backend-sa ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - namespace: default - name: orchestration-backend-role -rules: - - apiGroups: [""] # "" indicates the core API group - resources: ["pods"] - verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] - - apiGroups: [ "batch" ] # "" indicates the core API group - resources: [ "jobs" ] - verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: orchestration-backend-role - namespace: default -subjects: - - kind: ServiceAccount - name: orchestration-backend-sa - namespace: default -roleRef: - kind: Role - name: orchestration-backend-role - apiGroup: rbac.authorization.k8s.io diff --git a/kubernetes-route.yaml b/kubernetes-route.yaml deleted file mode 100644 index 992a100..0000000 --- a/kubernetes-route.yaml +++ /dev/null @@ -1,26 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: orchestration-backend-service -spec: - selector: - app: orchestration-backend - ports: - - protocol: TCP - port: 8080 ---- -apiVersion: traefik.containo.us/v1alpha1 -kind: IngressRoute -metadata: - name: orchestration-backend-route - namespace: default -spec: - entryPoints: - - websecure - routes: - - match: Host(`kubernetes.demo.nsidr.org`) && PathPrefix(`/translator`) - kind: Rule - services: - - name: orchestration-backend-service - namespace: default - port: 8080 \ No newline at end of file diff --git a/kubernetes.yaml b/kubernetes.yaml deleted file mode 100644 index e002423..0000000 --- a/kubernetes.yaml +++ /dev/null @@ -1,59 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: dissco-orchestration-backend - labels: - app: dissco-orchestration-backend -spec: - replicas: 1 - selector: - matchLabels: - app: dissco-orchestration-backend - template: - metadata: - labels: - app: dissco-orchestration-backend - spec: - serviceAccountName: dissco-orchestration-backend-sa - containers: - - name: dissco-orchestration-backend - image: public.ecr.aws/dissco/dissco-core-orchestration-backend:latest - imagePullPolicy: Always - ports: - - containerPort: 8080 - env: - - name: spring.datasource.url - value: jdbc:postgresql://database-1.cbppwfnjypll.eu-west-2.rds.amazonaws.com/dissco - - name: spring.datasource.username - valueFrom: - secretKeyRef: - name: db-user-pass - key: username - - name: spring.datasource.password - valueFrom: - secretKeyRef: - name: db-user-pass - key: password - - name: spring.security.oauth2.resourceserver.jwt.issuer-uri - value: https://login-demo.dissco.eu/auth/realms/dissco - - name: spring.security.oauth2.authorizationserver.endpoint.jwk-set-uri - value: ${spring.security.oauth2.resourceserver.jwt.issuer-uri}/protocol/openid-connect/certs - - name: jwt.auth.converter.resource-id - value: springboot-keycloak-client - - name: kafka.publisher.host - value: kafka.kafka.svc.cluster.local:9092 - - name: mas.kafkaHost - value: kafka.kafka.svc.cluster.local:9092 - - name: application.baseUrl - value: sandbox.dissco.tech - - name: token.id - value: demo-api-client - - name: token.grant-type - value: client_credentials - - name: endpoint.token-endpoint - value: https://login-demo.dissco.eu/auth/realms/dissco/protocol/openid-connect/token - - name: endpoint.handle-endpoint - value: https://sandbox.dissco.tech/handle-manager/api/v1/pids/ - securityContext: - runAsNonRoot: true - allowPrivilegeEscalation: false diff --git a/src/main/java/eu/dissco/orchestration/backend/properties/TranslatorJobProperties.java b/src/main/java/eu/dissco/orchestration/backend/properties/TranslatorJobProperties.java index 00f3040..8694f52 100644 --- a/src/main/java/eu/dissco/orchestration/backend/properties/TranslatorJobProperties.java +++ b/src/main/java/eu/dissco/orchestration/backend/properties/TranslatorJobProperties.java @@ -16,12 +16,12 @@ public class TranslatorJobProperties { private String image = "public.ecr.aws/dissco/dissco-core-translator:latest"; @NotBlank - private String kafkaHost = "kafka.kafka.svc.cluster.local:9092"; + private String kafkaHost = "kafka-cluster-kafka-bootstrap.kafka.svc.cluster.local:9092"; @NotBlank - private String kafkaTopic = "digital-specimen"; + private String kafkaTopic = "col"; @NotBlank - private String namespace = "default"; + private String namespace = "translator-services"; } diff --git a/src/main/java/eu/dissco/orchestration/backend/service/SourceSystemService.java b/src/main/java/eu/dissco/orchestration/backend/service/SourceSystemService.java index 7355edf..197e19c 100644 --- a/src/main/java/eu/dissco/orchestration/backend/service/SourceSystemService.java +++ b/src/main/java/eu/dissco/orchestration/backend/service/SourceSystemService.java @@ -36,6 +36,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Random; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.RandomStringUtils; @@ -59,6 +60,7 @@ public class SourceSystemService { private final TranslatorJobProperties jobProperties; private final Configuration configuration; private final BatchV1Api batchV1Api; + private final Random random; private static String getSuffix(String sourceSystemId) { return sourceSystemId.substring(sourceSystemId.indexOf('/') + 1).toLowerCase(); @@ -66,7 +68,7 @@ private static String getSuffix(String sourceSystemId) { private static String generateJobName(SourceSystemRecord sourceSystem, boolean isCron) { var name = - sourceSystem.sourceSystem().translatorType().getName().toLowerCase() + "-" + + sourceSystem.sourceSystem().translatorType().getLiteral().toLowerCase() + "-" + getSuffix(sourceSystem.id()) + "-translator-service"; if (!isCron) { name = name + "-" + RandomStringUtils.randomAlphabetic(6).toLowerCase(); @@ -336,6 +338,7 @@ private Map getTemplateProperties(SourceSystemRecord sourceSyste map.put("image", jobProperties.getImage()); map.put("sourceSystemId", sourceSystem.id()); map.put("jobName", jobName); + map.put("namespace", jobProperties.getNamespace()); map.put("containerName", jobName); map.put("kafkaHost", jobProperties.getKafkaHost()); map.put("kafkaTopic", jobProperties.getKafkaTopic()); @@ -346,8 +349,8 @@ private Map getTemplateProperties(SourceSystemRecord sourceSyste } private String generateCron() { - String day = RandomStringUtils.randomNumeric(0, 6); - String hour = RandomStringUtils.randomNumeric(0, 23); + String day = String.valueOf(random.nextInt(7)); + String hour = String.valueOf(random.nextInt(23)); return "0 " + hour + " * * " + day; } diff --git a/src/main/resources/templates/biocase-cron-job.ftl b/src/main/resources/templates/biocase-cron-job.ftl index fbc150e..f8470ce 100644 --- a/src/main/resources/templates/biocase-cron-job.ftl +++ b/src/main/resources/templates/biocase-cron-job.ftl @@ -2,6 +2,7 @@ apiVersion: batch/v1 kind: CronJob metadata: name: ${jobName} + namespace: ${namespace} spec: schedule: ${cron} jobTemplate: @@ -9,6 +10,7 @@ spec: template: spec: restartPolicy: Never + serviceAccountName: translator-secret-manager containers: - name: ${containerName} image: ${image} @@ -42,8 +44,19 @@ spec: value: https://doi.org/21.T11148/bbad8c4e101e8af01115 - name: fdo.digital-specimen-type value: https://doi.org/21.T11148/894b1e6cad57e921764e - - name: JAVA_OPTS - value: -server -XX:+useContainerSupport -XX:MaxRAMPercentage=75 --illegal-access=deny + - name: JAVA_TOOL_OPTIONS + value: -XX:MaxRAMPercentage=85 securityContext: runAsNonRoot: true allowPrivilegeEscalation: false + volumeMounts: + - name: db-secrets + mountPath: "/mnt/secrets-store/db-secrets" + readOnly: true + volumes: + - name: db-secrets + csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: "db-secrets" \ No newline at end of file diff --git a/src/main/resources/templates/biocase-translator-job.ftl b/src/main/resources/templates/biocase-translator-job.ftl index 7d30fe4..ea68990 100644 --- a/src/main/resources/templates/biocase-translator-job.ftl +++ b/src/main/resources/templates/biocase-translator-job.ftl @@ -2,11 +2,13 @@ apiVersion: batch/v1 kind: Job metadata: name: ${jobName} + namespace: ${namespace} spec: backoffLimit: 2 template: spec: restartPolicy: Never + serviceAccountName: translator-secret-manager containers: - name: ${containerName} image: ${image} @@ -40,8 +42,19 @@ spec: value: https://doi.org/21.T11148/bbad8c4e101e8af01115 - name: fdo.digital-specimen-type value: https://doi.org/21.T11148/894b1e6cad57e921764e - - name: JAVA_OPTS - value: -server -XX:+useContainerSupport -XX:MaxRAMPercentage=75 --illegal-access=deny + - name: JAVA_TOOL_OPTIONS + value: -XX:MaxRAMPercentage=85 securityContext: runAsNonRoot: true allowPrivilegeEscalation: false + volumeMounts: + - name: db-secrets + mountPath: "/mnt/secrets-store/db-secrets" + readOnly: true + volumes: + - name: db-secrets + csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: "db-secrets" diff --git a/src/main/resources/templates/dwca-cron-job.ftl b/src/main/resources/templates/dwca-cron-job.ftl index 5db51d8..8d2fcd6 100644 --- a/src/main/resources/templates/dwca-cron-job.ftl +++ b/src/main/resources/templates/dwca-cron-job.ftl @@ -2,12 +2,14 @@ apiVersion: batch/v1 kind: CronJob metadata: name: ${jobName} + namespace: ${namespace} spec: schedule: ${cron} jobTemplate: spec: template: spec: + serviceAccountName: translator-secret-manager restartPolicy: Never containers: - name: ${containerName} @@ -46,14 +48,23 @@ spec: value: https://doi.org/21.T11148/bbad8c4e101e8af01115 - name: fdo.digital-specimen-type value: https://doi.org/21.T11148/894b1e6cad57e921764e - - name: JAVA_OPTS - value: -server -XX:+useContainerSupport -XX:MaxRAMPercentage=75 --illegal-access=deny + - name: JAVA_TOOL_OPTIONS + value: -XX:MaxRAMPercentage=85 securityContext: runAsNonRoot: true allowPrivilegeEscalation: false volumeMounts: - - mountPath: /temp - name: temp-volume + - mountPath: /temp + name: temp-volume + - name: db-secrets + mountPath: "/mnt/secrets-store/db-secrets" + readOnly: true volumes: - - name: temp-volume - emptyDir: { } \ No newline at end of file + - name: temp-volume + emptyDir: { } + - name: db-secrets + csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: "db-secrets" \ No newline at end of file diff --git a/src/main/resources/templates/dwca-translator-job.ftl b/src/main/resources/templates/dwca-translator-job.ftl index f866405..9c51c46 100644 --- a/src/main/resources/templates/dwca-translator-job.ftl +++ b/src/main/resources/templates/dwca-translator-job.ftl @@ -2,11 +2,13 @@ apiVersion: batch/v1 kind: Job metadata: name: ${jobName} + namespace: ${namespace} spec: backoffLimit: 2 template: spec: restartPolicy: Never + serviceAccountName: translator-secret-manager containers: - name: ${containerName} image: ${image} @@ -44,14 +46,23 @@ spec: value: https://doi.org/21.T11148/bbad8c4e101e8af01115 - name: fdo.digital-specimen-type value: https://doi.org/21.T11148/894b1e6cad57e921764e - - name: JAVA_OPTS - value: -server -XX:+useContainerSupport -XX:MaxRAMPercentage=75 --illegal-access=deny + - name: JAVA_TOOL_OPTIONS + value: -XX:MaxRAMPercentage=85 securityContext: runAsNonRoot: true allowPrivilegeEscalation: false volumeMounts: - mountPath: /temp name: temp-volume + - name: db-secrets + mountPath: "/mnt/secrets-store/db-secrets" + readOnly: true volumes: - name: temp-volume - emptyDir: { } \ No newline at end of file + emptyDir: { } + - name: db-secrets + csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: "db-secrets" diff --git a/src/test/java/eu/dissco/orchestration/backend/service/SourceSystemServiceTest.java b/src/test/java/eu/dissco/orchestration/backend/service/SourceSystemServiceTest.java index 87c195f..19f961d 100644 --- a/src/test/java/eu/dissco/orchestration/backend/service/SourceSystemServiceTest.java +++ b/src/test/java/eu/dissco/orchestration/backend/service/SourceSystemServiceTest.java @@ -60,6 +60,7 @@ import java.util.Collections; import java.util.List; import java.util.Optional; +import java.util.Random; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -71,7 +72,7 @@ @ExtendWith(MockitoExtension.class) class SourceSystemServiceTest { - private static final String NAMESPACE = "default"; + private static final String NAMESPACE = "translator-services"; private final ObjectMapper yamlMapper = new ObjectMapper( new YAMLFactory()).findAndRegisterModules(); @@ -92,6 +93,7 @@ class SourceSystemServiceTest { private MappingService mappingService; @Mock private BatchV1Api batchV1Api; + private Random random = new Random(); private MockedStatic mockedStatic; @@ -100,7 +102,7 @@ class SourceSystemServiceTest { @BeforeEach void setup() throws IOException { service = new SourceSystemService(builder, handleComponent, repository, mappingService, - kafkaPublisherService, MAPPER, yamlMapper, jobProperties, configuration, batchV1Api); + kafkaPublisherService, MAPPER, yamlMapper, jobProperties, configuration, batchV1Api, random); initTime(); initFreeMaker(); }