Skip to content

Commit

Permalink
Merge pull request #53 from DiSSCo/hotfix/mas-secret-update
Browse files Browse the repository at this point in the history 
mas service account
  • Loading branch information
@southeo
southeo authored Sep 18, 2024
2 parents 2c0b7bc + f927f48 commit ada4ed5
Show file tree
Hide file tree
Showing 3 changed files with 44 additions and 46 deletions.
68 changes: 31 additions & 37 deletions src/main/java/eu/dissco/orchestration/backend/service/MachineAnnotationServiceService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,7 @@ public JsonApiWrapper createMachineAnnotationService(
var mas = buildMachineAnnotationService(masRequest, 1, userId, handle,
Instant.now());
repository.createMachineAnnotationService(mas);
createDeployment(mas, masRequest);
createDeployment(mas);
publishCreateEvent(mas);
return wrapSingleResponse(mas, path);
} catch (PidException e) {
Expand Down Expand Up @@ -166,12 +166,11 @@ private void setDefaultMas(MachineAnnotationServiceRequest mas, String handle) {
}
}

private void createDeployment(MachineAnnotationService mas,
MachineAnnotationServiceRequest masRequest)
private void createDeployment(MachineAnnotationService mas)
throws ProcessingFailedException {
var successfulDeployment = false;
try {
successfulDeployment = deployMasToCluster(mas, true, masRequest);
successfulDeployment = deployMasToCluster(mas, true);
deployKedaToCluster(mas);
} catch (KubernetesFailedException e) {
rollbackMasCreation(mas, successfulDeployment, false);
Expand All @@ -198,17 +197,12 @@ private void deployKedaToCluster(MachineAnnotationService mas)
}

private boolean deployMasToCluster(MachineAnnotationService mas,
boolean create, MachineAnnotationServiceRequest masRequest)
boolean create)
throws KubernetesFailedException {
var shortPid = getName(mas.getId());
try {
V1Deployment deployment;
if (masRequest == null) { // Null only when we're deploying a previous version
deployment = appsV1Api.readNamespacedDeployment(properties.getNamespace(),
getName((mas.getId()) + DEPLOYMENT)).execute();
} else {
deployment = getV1Deployment(mas, shortPid, masRequest);
}
deployment = getV1Deployment(mas, shortPid);
if (create) {
appsV1Api.createNamespacedDeployment(properties.getNamespace(), deployment).execute();
} else {
Expand All @@ -226,11 +220,10 @@ private boolean deployMasToCluster(MachineAnnotationService mas,
return true;
}

private V1Deployment getV1Deployment(MachineAnnotationService mas, String shortPid,
MachineAnnotationServiceRequest masRequest)
private V1Deployment getV1Deployment(MachineAnnotationService mas, String shortPid)
throws IOException, TemplateException {
var templateProperties = getDeploymentTemplateProperties(mas, shortPid);
var deploymentString = fillDeploymentTemplate(templateProperties, masRequest);
var deploymentString = fillDeploymentTemplate(templateProperties, mas);
return mapper.readValue(deploymentString, V1Deployment.class);
}

Expand Down Expand Up @@ -271,13 +264,13 @@ private Map<String, Object> getDeploymentTemplateProperties(MachineAnnotationSer
return map;
}

private List<JsonNode> addMasKeys(MachineAnnotationServiceRequest masRequest) {
private List<JsonNode> addMasKeys(MachineAnnotationService mas) {
var keyNode = new ArrayList<JsonNode>();
if (masRequest == null) {
if (mas == null) {
return List.of(mapper.createObjectNode());
}
if (masRequest.getOdsHasEnvironment() != null) {
masRequest.getOdsHasEnvironment().forEach(env -> {
if (mas.getOdsHasEnvironment() != null) {
mas.getOdsHasEnvironment().forEach(env -> {
if (env.getValue() instanceof String stringVal) {
keyNode.add(mapper.createObjectNode()
.put(NAME, env.getName())
Expand All @@ -295,8 +288,8 @@ private List<JsonNode> addMasKeys(MachineAnnotationServiceRequest masRequest) {
}
});
}
if (masRequest.getOdsHasSecret() != null) {
masRequest.getOdsHasSecret().forEach(secret -> keyNode.add(mapper.createObjectNode()
if (mas.getOdsHasSecret() != null) {
mas.getOdsHasSecret().forEach(secret -> keyNode.add(mapper.createObjectNode()
.put(NAME, secret.getSecretName())
.set("valueFrom", mapper.createObjectNode()
.set("secretKeyRef", mapper.createObjectNode()
Expand All @@ -307,14 +300,14 @@ private List<JsonNode> addMasKeys(MachineAnnotationServiceRequest masRequest) {
}

private String fillDeploymentTemplate(Map<String, Object> templateProperties,
MachineAnnotationServiceRequest masRequest)
MachineAnnotationService mas)
throws IOException, TemplateException {
var writer = new StringWriter();
deploymentTemplate.process(templateProperties, writer);
var templateAsNode = (ObjectNode) mapper.readTree(writer.toString());
var defaultKeyNode = (ArrayNode) templateAsNode.get("spec").get("template").get("spec")
.get("containers").get(0).get("env");
defaultKeyNode.addAll(addMasKeys(masRequest));
defaultKeyNode.addAll(addMasKeys(mas));
return mapper.writeValueAsString(templateAsNode);
}

Expand Down Expand Up @@ -378,8 +371,8 @@ public JsonApiWrapper updateMachineAnnotationService(String id,
return null;
} else {
repository.updateMachineAnnotationService(machineAnnotationService);
updateDeployment(machineAnnotationService, currentMas, masRequest);
publishUpdateEvent(machineAnnotationService, currentMas, masRequest);
updateDeployment(machineAnnotationService, currentMas);
publishUpdateEvent(machineAnnotationService, currentMas);
return wrapSingleResponse(machineAnnotationService, path);
}
} else {
Expand Down Expand Up @@ -412,14 +405,14 @@ private boolean isEqual(MachineAnnotationService mas, MachineAnnotationService c
}

private void updateDeployment(MachineAnnotationService mas,
MachineAnnotationService currentMas, MachineAnnotationServiceRequest masRequest)
MachineAnnotationService currentMas)
throws ProcessingFailedException {
var successfulDeployment = false;
try {
successfulDeployment = deployMasToCluster(mas, false, masRequest);
successfulDeployment = deployMasToCluster(mas, false);
updateKedaResource(mas, currentMas);
} catch (KubernetesFailedException e) {
rollbackToPreviousVersion(currentMas, successfulDeployment, false, masRequest);
rollbackToPreviousVersion(currentMas, successfulDeployment, false);
throw new ProcessingFailedException("Failed to update kubernetes resources", e);
}
}
Expand Down Expand Up @@ -463,28 +456,27 @@ private void updateKedaResource(MachineAnnotationService mas,
}

private void publishUpdateEvent(MachineAnnotationService mas,
MachineAnnotationService currentMas, MachineAnnotationServiceRequest masRequest)
MachineAnnotationService currentMas)
throws ProcessingFailedException {
try {
kafkaPublisherService.publishUpdateEvent(mapper.valueToTree(mas),
mapper.valueToTree(currentMas));
} catch (JsonProcessingException e) {
log.error("Unable to publish message to Kafka", e);
rollbackToPreviousVersion(currentMas, true, true, masRequest);
rollbackToPreviousVersion(currentMas, true, true);
throw new ProcessingFailedException("Failed to create new machine annotation service", e);
}
}

private void rollbackToPreviousVersion(MachineAnnotationService currentMas,
boolean rollbackDeployment, boolean rollbackKeda,
MachineAnnotationServiceRequest masRequest)
boolean rollbackDeployment, boolean rollbackKeda)
throws ProcessingFailedException {
repository.updateMachineAnnotationService(currentMas);
if (rollbackDeployment) {
try {
log.warn(
"Rolling back to previous version of kubernetes deployment with environmental variables provided in update request. Environment may be out of sync with deployment");
deployMasToCluster(currentMas, false, masRequest);
deployMasToCluster(currentMas, false);
} catch (KubernetesFailedException e) {
log.error(
"Fatal exception, unable to rollback kubernetes deployment for: {} ",
Expand Down Expand Up @@ -513,10 +505,12 @@ public void tombstoneMachineAnnotationService(String id, Agent agent)
var tombstoneMas = buildTombstoneMachineAnnotationService(mas, agent, timestamp);
repository.tombstoneMachineAnnotationService(tombstoneMas, timestamp);
try {
kafkaPublisherService.publishTombstoneEvent(mapper.valueToTree(tombstoneMas), mapper.valueToTree(mas));
} catch (JsonProcessingException e){
kafkaPublisherService.publishTombstoneEvent(mapper.valueToTree(tombstoneMas),
mapper.valueToTree(mas));
} catch (JsonProcessingException e) {
log.error("Unable to publish tombstone event to provenance service", e);
throw new ProcessingFailedException("Unable to publish tombstone event to provenance service", e);
throw new ProcessingFailedException(
"Unable to publish tombstone event to provenance service", e);
}
} else {
throw new NotFoundException("Requested machine annotation system: " + id + "does not exist");
Expand All @@ -527,7 +521,7 @@ private void tombstoneHandle(String handle) throws ProcessingFailedException {
var request = fdoRecordService.buildTombstoneRequest(ObjectType.MAS, handle);
try {
handleComponent.tombstoneHandle(request, handle);
} catch (PidException e){
} catch (PidException e) {
log.error("Unable to tombstone handle {}", handle, e);
throw new ProcessingFailedException("Unable to tombstone handle", e);
}
Expand Down Expand Up @@ -591,7 +585,7 @@ private void deleteDeployment(MachineAnnotationService currentMas)
"Deletion of kubernetes keda failed for record: {}, with code: {} and message: {}",
currentMas, e.getCode(), e.getResponseBody());
try {
deployMasToCluster(currentMas, true, null);
deployMasToCluster(currentMas, true);
} catch (KubernetesFailedException ex) {
log.error("Failed error, unable to create deployment after failed keda deletion");
}
Expand Down
2 changes: 2 additions & 0 deletions src/main/resources/templates/mas-template.ftl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@
}
},
"spec": {
"serviceAccountName":"mas-secret-manager",
"automountServiceAccountToken": true,
"containers": [
{
"name": "${pid}",
Expand Down
20 changes: 11 additions & 9 deletions ...est/java/eu/dissco/orchestration/backend/service/MachineAnnotationServiceServiceTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,6 @@
import io.kubernetes.client.openapi.apis.AppsV1Api;
import io.kubernetes.client.openapi.apis.AppsV1Api.APIcreateNamespacedDeploymentRequest;
import io.kubernetes.client.openapi.apis.AppsV1Api.APIdeleteNamespacedDeploymentRequest;
import io.kubernetes.client.openapi.apis.AppsV1Api.APIreadNamespacedDeploymentRequest;
import io.kubernetes.client.openapi.apis.AppsV1Api.APIreplaceNamespacedDeploymentRequest;
import io.kubernetes.client.openapi.apis.CustomObjectsApi;
import io.kubernetes.client.openapi.apis.CustomObjectsApi.APIcreateNamespacedCustomObjectRequest;
Expand Down Expand Up @@ -660,11 +659,14 @@ void testTombstoneMasKafkaFailed() throws Exception {
var deleteCustom = mock(APIdeleteNamespacedCustomObjectRequest.class);
given(customObjectsApi.deleteNamespacedCustomObject("keda.sh", "v1alpha1", "namespace",
"scaledobjects", "gw0-pop-xsl-scaled-object")).willReturn(deleteCustom);
doThrow(JsonProcessingException.class).when(kafkaPublisherService).publishTombstoneEvent(any(), any());
given(fdoRecordService.buildTombstoneRequest(ObjectType.MAS, BARE_HANDLE)).willReturn(givenTombstoneRequestMas());
doThrow(JsonProcessingException.class).when(kafkaPublisherService)
.publishTombstoneEvent(any(), any());
given(fdoRecordService.buildTombstoneRequest(ObjectType.MAS, BARE_HANDLE)).willReturn(
givenTombstoneRequestMas());

// When
assertThrowsExactly(ProcessingFailedException.class, () -> service.tombstoneMachineAnnotationService(BARE_HANDLE, givenAgent()));
assertThrowsExactly(ProcessingFailedException.class,
() -> service.tombstoneMachineAnnotationService(BARE_HANDLE, givenAgent()));

// Then
then(repository).should().tombstoneMachineAnnotationService(any(), any());
Expand Down Expand Up @@ -701,15 +703,13 @@ void testDeleteKedaFails() throws ApiException {
given(repository.getActiveMachineAnnotationService(BARE_HANDLE)).willReturn(
Optional.of(givenMas()));
given(properties.getNamespace()).willReturn("namespace");
given(properties.getKafkaHost()).willReturn("kafka.svc.cluster.local:9092");
var createDeploy = mock(APIcreateNamespacedDeploymentRequest.class);
given(appsV1Api.createNamespacedDeployment(eq("namespace"), any(V1Deployment.class)))
.willReturn(createDeploy);
var deleteDeploy = mock(APIdeleteNamespacedDeploymentRequest.class);
given(appsV1Api.deleteNamespacedDeployment(SUFFIX.toLowerCase() + "-deployment",
"namespace")).willReturn(deleteDeploy);
var deleteRead = mock(APIreadNamespacedDeploymentRequest.class);
given(appsV1Api.readNamespacedDeployment(any(), anyString())).willReturn(deleteRead);
given(deleteRead.execute()).willReturn(mock(V1Deployment.class));
var deleteCustom = mock(APIdeleteNamespacedCustomObjectRequest.class);
given(customObjectsApi.deleteNamespacedCustomObject("keda.sh", "v1alpha1", "namespace",
"scaledobjects", "gw0-pop-xsl-scaled-object")).willReturn(deleteCustom);
Expand All @@ -735,8 +735,10 @@ private static Stream<Arguments> masKeys() {
return Stream.of(
Arguments.of(null, null),
Arguments.of(givenMasEnvironment(), givenMasSecrets()),
Arguments.of(List.of(new MachineAnnotationServiceEnvironment("name", 1)), givenMasSecrets()),
Arguments.of(List.of(new MachineAnnotationServiceEnvironment("name", true)), givenMasSecrets())
Arguments.of(List.of(new MachineAnnotationServiceEnvironment("name", 1)),
givenMasSecrets()),
Arguments.of(List.of(new MachineAnnotationServiceEnvironment("name", true)),
givenMasSecrets())
);
}

Expand Down

0 comments on commit ada4ed5

Please sign in to comment.