Skip to content

Commit

Permalink
Add single Chronos jobs status update
Browse files Browse the repository at this point in the history 
See #34
  • Loading branch information
@lorenzo-biava
lorenzo-biava committed May 23, 2016
1 parent 66f2ca3 commit 4b2ecb7
Show file tree
Hide file tree
Showing 3 changed files with 93 additions and 44 deletions.
8 changes: 2 additions & 6 deletions ...it/reply/orchestrator/service/deployment/providers/AbstractDeploymentProviderService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ public void updateOnError(String deploymentUuid, Throwable throwable) {
* Update the status of the deployment with an error message.
*
* @param deploymentUuid
* the deplyment id
* the deployment id
* @param message
* the error message
*/
Expand All @@ -126,21 +126,17 @@ public void updateOnError(String deploymentUuid, String message) {
break;
case CREATE_IN_PROGRESS:
deployment.setStatus(Status.CREATE_FAILED);
updateResources(deployment, Status.CREATE_FAILED);
break;
case DELETE_IN_PROGRESS:
deployment.setStatus(Status.DELETE_FAILED);
updateResources(deployment, Status.DELETE_FAILED);
break;
case UPDATE_IN_PROGRESS:
deployment.setStatus(Status.UPDATE_FAILED);
updateResources(deployment, Status.UPDATE_FAILED);
break;
default:
LOG.error("updateOnError: unsupported deployment status: {}. Setting status to {}",
deployment.getStatus(), Status.UNKNOWN.toString());
deployment.setStatus(Status.UNKNOWN);
updateResources(deployment, Status.UNKNOWN);
break;
}
deployment.setTask(Task.NONE);
Expand Down Expand Up @@ -184,7 +180,7 @@ public void updateOnSuccess(String deploymentUuid) {
}
}

private void updateResources(Deployment deployment, Status status) {
protected void updateResources(Deployment deployment, Status status) {

for (Resource resource : deployment.getResources()) {
if (status.equals(Status.CREATE_COMPLETE) || status.equals(Status.UPDATE_COMPLETE)) {
Expand Down
86 changes: 48 additions & 38 deletions src/main/java/it/reply/orchestrator/service/deployment/providers/ChronosServiceImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import it.reply.orchestrator.dal.entity.Resource;
import it.reply.orchestrator.dal.repository.ResourceRepository;
import it.reply.orchestrator.enums.DeploymentProvider;
import it.reply.orchestrator.enums.NodeStates;
import it.reply.orchestrator.enums.Task;
import it.reply.orchestrator.exception.OrchestratorException;
import it.reply.orchestrator.exception.service.DeploymentException;
Expand Down Expand Up @@ -141,7 +142,7 @@ public boolean doDeploy(Deployment deployment) {

// Create Jobs in the required order on Chronos
LOG.debug("Launching jobs for deployment <{}> on Chronos", deployment.getId());
createJobsOnChronos(jobgraph, getChronosClient());
createJobsOnChronos(deployment, jobgraph, getChronosClient());

return true;
} catch (RuntimeException exception) { // Chronos job launch error
Expand All @@ -157,8 +158,8 @@ public boolean doDeploy(Deployment deployment) {
}
}

protected void createJobsOnChronos(Multimap<JobDependencyType, IndigoJob> jobgraph,
Chronos client) {
protected void createJobsOnChronos(Deployment deployment,
Multimap<JobDependencyType, IndigoJob> jobgraph, Chronos client) {
// Create Jobs in the required order on Chronos

IndigoJob currentJob = null;
Expand All @@ -182,9 +183,13 @@ protected void createJobsOnChronos(Multimap<JobDependencyType, IndigoJob> jobgra

LOG.debug("Created job on Chronos: name <{}>, {}", currentJob.getChronosJob().getName(),
nodeTypeMsg);
// Update job status
updateResource(deployment, currentJob, NodeStates.CREATED);
}

} catch (ChronosException exception) { // Chronos job launch error
// Update job status
updateResource(deployment, currentJob, NodeStates.ERROR);
// TODO use a custom exception ?
throw new RuntimeException(
String.format("Failed to launch job <%s> on Chronos. Status Code: <%s>",
Expand Down Expand Up @@ -259,26 +264,44 @@ public boolean isDeployed(Deployment deployment) throws DeploymentException {
String jobName = job.getChronosJob().getName();
Job updatedJob = getJobStatus(client, jobName);
if (updatedJob == null) {
String errorMsg =
String.format("Failed to deploy deployment <%s>. Chronos job <%s> does not exist",
deployment.getId(), jobName);
String errorMsg = String.format(
"Failed to deploy deployment <%s>. Chronos job <%s> (id: <%s>) does not exist",
deployment.getId(), job.getToscaNodeName(), jobName);
LOG.error(errorMsg);
// Update job status
updateResource(deployment, job, NodeStates.ERROR);
throw new DeploymentException(errorMsg);
}

JobState jobState = getLastState(updatedJob);
LOG.debug("Status for Chronos job <{}> is <{}>", jobName, jobState);

// Go ahead only if the job succeeded
if (!checkJobState(deployment.getId(), jobName, jobState)) {
return false;
if (jobState != JobState.SUCCESS) {
if (jobState != JobState.FAILURE) {
// Job still in progress
return false;
} else {
// Job failed -> Deployment failed!
String errorMsg = String.format(
"Failed to deploy deployment <%s>. Chronos job <%s> (id: <%s>) status is <%s>",
deployment.getId(), job.getToscaNodeName(), jobName, jobState);
LOG.error(errorMsg);
// Update job status
updateResource(deployment, job, NodeStates.ERROR);
throw new DeploymentException(errorMsg);
}
} else {
// Update job status
updateResource(deployment, job, NodeStates.STARTED);
}
}

// Here all task succeeded -> deployment is ready
return true;
} catch (DeploymentException dex) {
// Deploy failed; let caller know (as for the method definitiion)
// Deploy failed; let caller know (as for the method definition)
updateOnError(deployment.getId(), dex);
throw dex;
} catch (RuntimeException ex) {
LOG.error("Failed to update deployment <{}>", ex);
Expand All @@ -291,32 +314,13 @@ public boolean isDeployed(Deployment deployment) throws DeploymentException {

}

/**
*
* @param deploymentId
* @param jobName
* @param jobState
* @return <tt>true</tt> if the job succeeded, <tt>false</tt> if the job is still in progress.
* @throws DeploymentException
* if the job failed.
*/
protected boolean checkJobState(String deploymentId, String jobName, JobState jobState)
throws DeploymentException {
if (jobState == JobState.SUCCESS) {
return true;
}
private void updateResource(Deployment deployment, IndigoJob job, NodeStates state) {

if (jobState != JobState.FAILURE) {
// Job still in progress
return false;
} else {
// Job failed -> Deployment failed!
String errorMsg =
String.format("Failed to deploy deployment <%s>. Chronos job <%s> status is <%s>",
deploymentId, jobName, jobState);
LOG.error(errorMsg);
throw new DeploymentException(errorMsg);
}
// Find the Resource from DB
Resource resource = resourceRepository
.findByToscaNodeNameAndDeployment_id(job.getToscaNodeName(), deployment.getId());
resource.setState(state);
// resourceRepository.save(resource);
}

/**
Expand Down Expand Up @@ -416,9 +420,13 @@ public boolean doUndeploy(Deployment deployment) {
Multimap<JobDependencyType, IndigoJob> jobgraph =
generateJobGraph(deployment, generateStubOneData());

for (Resource resource : deployment.getResources()) {
resource.setState(NodeStates.DELETING);
}

// Create Jobs in the required order on Chronos
LOG.debug("Deleting jobs for deployment <{}> on Chronos", deployment.getId());
deleteJobsOnChronos(jobgraph, getChronosClient(), true);
deleteJobsOnChronos(deployment, jobgraph, getChronosClient(), true);

return true;
} catch (RuntimeException exception) { // Chronos job launch error
Expand Down Expand Up @@ -447,8 +455,8 @@ public boolean doUndeploy(Deployment deployment) {
* tries to delete every other job.
* @return <tt>true</tt> if all jobs have been deleted, <tt>false</tt> otherwise.
*/
protected boolean deleteJobsOnChronos(Multimap<JobDependencyType, IndigoJob> jobgraph,
Chronos client, boolean failAtFirst) {
protected boolean deleteJobsOnChronos(Deployment deployment,
Multimap<JobDependencyType, IndigoJob> jobgraph, Chronos client, boolean failAtFirst) {

IndigoJob currentJob = null;
boolean failed = false;
Expand All @@ -467,6 +475,9 @@ protected boolean deleteJobsOnChronos(Multimap<JobDependencyType, IndigoJob> job
LOG.error(errorMsg);
failed = true;

// Update job status
updateResource(deployment, currentJob, NodeStates.ERROR);

// Only throw exception in required
if (failAtFirst) {
// TODO use a custom exception ?
Expand All @@ -476,7 +487,6 @@ protected boolean deleteJobsOnChronos(Multimap<JobDependencyType, IndigoJob> job
}

return !failed;

}

public static class IndigoJob {
Expand Down
43 changes: 43 additions & 0 deletions src/main/java/it/reply/orchestrator/service/deployment/providers/ImServiceImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import it.reply.orchestrator.dal.repository.ResourceRepository;
import it.reply.orchestrator.enums.DeploymentProvider;
import it.reply.orchestrator.enums.NodeStates;
import it.reply.orchestrator.enums.Status;
import it.reply.orchestrator.enums.Task;
import it.reply.orchestrator.exception.OrchestratorException;
import it.reply.orchestrator.exception.service.DeploymentException;
Expand Down Expand Up @@ -557,4 +558,46 @@ private ResponseError getImResponseError(ImClientErrorException exception) {
private void logImErrorResponse(ImClientErrorException exception) {
LOG.error(exception.getResponseError().getFormattedErrorMessage());
}

// FIXME Remove once IM handles single nodes state update
/**
* Update the status of the deployment with an error message.
*
* @param deploymentUuid
* the deployment id
* @param message
* the error message
*/
public void updateOnError(String deploymentUuid, String message) {
Deployment deployment = deploymentRepository.findOne(deploymentUuid);
switch (deployment.getStatus()) {
case CREATE_FAILED:
case UPDATE_FAILED:
case DELETE_FAILED:
LOG.warn("Deployment < {} > was already in {} state.", deploymentUuid,
deployment.getStatus());
break;
case CREATE_IN_PROGRESS:
deployment.setStatus(Status.CREATE_FAILED);
updateResources(deployment, Status.CREATE_FAILED);
break;
case DELETE_IN_PROGRESS:
deployment.setStatus(Status.DELETE_FAILED);
updateResources(deployment, Status.DELETE_FAILED);
break;
case UPDATE_IN_PROGRESS:
deployment.setStatus(Status.UPDATE_FAILED);
updateResources(deployment, Status.UPDATE_FAILED);
break;
default:
LOG.error("updateOnError: unsupported deployment status: {}. Setting status to {}",
deployment.getStatus(), Status.UNKNOWN.toString());
deployment.setStatus(Status.UNKNOWN);
updateResources(deployment, Status.UNKNOWN);
break;
}
deployment.setTask(Task.NONE);
deployment.setStatusReason(message);
deploymentRepository.save(deployment);
}
}

0 comments on commit 4b2ecb7

Please sign in to comment.