Skip to content

Commit

Permalink
Add template hard-coded OneData params replacement
Browse files Browse the repository at this point in the history
OneData properties are temporarily loaded from properties file

See #34
  • Loading branch information
lorenzo-biava committed May 19, 2016
1 parent d2dd1d7 commit 8e3c6c8
Show file tree
Hide file tree
Showing 4 changed files with 155 additions and 44 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,8 @@ public Deployment createDeployment(DeploymentRequest request) {

nodes = parsingResult.getTopology().getNodeTemplates();

// FIXME: Define function to decide DeploymentProvider (Temporary - just for prototyping)
isChronosDeployment = isChronosDeployment(nodes);
if (!isChronosDeployment) {
// FIXME (BAD HACK) IM templates need some parameters to be added, but regenerating the
// template string with the current library is risky (loses some information!!)
Expand All @@ -115,8 +117,7 @@ public Deployment createDeployment(DeploymentRequest request) {
Map<String, Object> params = new HashMap<>();
params.put("DEPLOYMENT_ID", deployment.getId());

// FIXME: Define function to decide DeploymentProvider (Temporary - just for prototyping)
isChronosDeployment = isChronosDeployment(nodes);
// FIXME Put in deployment provider field
params.put(WF_PARAM_DEPLOYMENT_TYPE,
(isChronosDeployment ? DEPLOYMENT_TYPE_CHRONOS : DEPLOYMENT_TYPE_TOSCA));

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,11 @@
import alien4cloud.model.components.DeploymentArtifact;
import alien4cloud.model.components.ListPropertyValue;
import alien4cloud.model.components.PropertyValue;
import alien4cloud.model.components.ScalarPropertyValue;
import alien4cloud.model.topology.Capability;
import alien4cloud.model.topology.NodeTemplate;
import alien4cloud.model.topology.RelationshipTemplate;
import alien4cloud.tosca.model.ArchiveRoot;
import alien4cloud.tosca.normative.SizeType;
import alien4cloud.tosca.parser.ParsingException;

Expand Down Expand Up @@ -61,20 +63,21 @@ public class ChronosServiceImpl extends AbstractDeploymentProviderService
@Autowired
ToscaService toscaService;

@Autowired
private ResourceRepository resourceRepository;

/**
* Temporary method to load Chronos connection properties from file (<b>just for experimental
* purpose</b>).
* Temporary method to load properties from file (<b>just for experimental purpose</b>).
*
* @return the properties.
*/
public Properties readChronosProperties() {
String path = "chronos.properties";
public Properties readProperties(String path) {
final Properties properties = new Properties();
try (InputStream in = getClass().getClassLoader().getResourceAsStream(path)) {
properties.load(in);
} catch (Exception ex) {
throw new RuntimeException(
"Failed to load properties file ('" + path + "') in 'test/resources' folder");
"Failed to load properties file ('" + path + "') in classpath 'resources' folder");
}
return properties;
}
Expand All @@ -86,7 +89,7 @@ public Properties readChronosProperties() {
* @return the Chronos client.
*/
public Chronos getChronosClient() {
Properties properties = readChronosProperties();
Properties properties = readProperties("chronos.properties");
String endpoint = properties.getProperty("endpoint");
String username = properties.getProperty("username");
String password = properties.getProperty("password");
Expand All @@ -98,6 +101,23 @@ public Chronos getChronosClient() {
return client;
}

/**
* Temporary method to generate default OneData settings.
*
* @return the {@link OneData} settings.
*/
protected OneData generateStubOneData() {
Properties properties = readProperties("onedata.properties");
String token = properties.getProperty("token");
String space = properties.getProperty("space");
String path = properties.getProperty("path");
String provider = properties.getProperty("provider");

LOG.info(String.format("Generating OneData settings with parameters: %s", properties));

return new OneData(token, space, path, provider);
}

public Collection<Job> getJobs(Chronos client) {
return client.getJobs();
}
Expand All @@ -117,7 +137,8 @@ public boolean doDeploy(Deployment deployment) {

// Generate INDIGOJob graph
LOG.debug("Generating job graph for deployment <{}>", deployment.getId());
Multimap<JobDependencyType, IndigoJob> jobgraph = generateJobGraph(deployment);
Multimap<JobDependencyType, IndigoJob> jobgraph =
generateJobGraph(deployment, generateStubOneData());

// Create Jobs in the required order on Chronos
LOG.debug("Launching jobs for deployment <{}> on Chronos", deployment.getId());
Expand Down Expand Up @@ -223,7 +244,8 @@ public boolean isDeployed(Deployment deployment) throws DeploymentException {
// Generate INDIGOJob graph
// FIXME: Do not regenerate every time (just for prototyping!)
LOG.debug("Generating job graph for deployment <{}>", deployment.getId());
Multimap<JobDependencyType, IndigoJob> jobgraph = generateJobGraph(deployment);
Multimap<JobDependencyType, IndigoJob> jobgraph =
generateJobGraph(deployment, generateStubOneData());

Chronos client = getChronosClient();

Expand Down Expand Up @@ -375,7 +397,8 @@ public boolean doUndeploy(Deployment deployment) {
// Generate INDIGOJob graph
// FIXME: Do not regenerate every time (just for prototyping!)
LOG.debug("Generating job graph for deployment <{}>", deployment.getId());
Multimap<JobDependencyType, IndigoJob> jobgraph = generateJobGraph(deployment);
Multimap<JobDependencyType, IndigoJob> jobgraph =
generateJobGraph(deployment, generateStubOneData());

// Create Jobs in the required order on Chronos
LOG.debug("Deleting jobs for deployment <{}> on Chronos", deployment.getId());
Expand Down Expand Up @@ -473,9 +496,6 @@ public String toString() {

}

@Autowired
private ResourceRepository resourceRepository;

/**
* Creates the {@link INDIGOJob} graph based on the given {@link Deployment} (the TOSCA template
* is parsed).
Expand All @@ -484,16 +504,22 @@ public String toString() {
* the input deployment.
* @return the job graph.
*/
protected Multimap<JobDependencyType, IndigoJob> generateJobGraph(Deployment deployment) {
protected Multimap<JobDependencyType, IndigoJob> generateJobGraph(Deployment deployment,
OneData odParams) {
String deploymentId = deployment.getId();
Multimap<JobDependencyType, IndigoJob> jobGraph = HashMultimap.create();
Map<String, IndigoJob> jobs = new HashMap<String, ChronosServiceImpl.IndigoJob>();

// Parse TOSCA template
Map<String, NodeTemplate> nodes = null;
try {
nodes = toscaService.getArchiveRootFromTemplate(deployment.getTemplate()).getResult()
.getTopology().getNodeTemplates();
String customizedTemplate = deployment.getTemplate();
// FIXME TEMPORARY - Replace hard-coded properties in nodes
customizedTemplate = replaceHardCodedParams(customizedTemplate, odParams);

ArchiveRoot ar = toscaService.getArchiveRootFromTemplate(customizedTemplate).getResult();

nodes = ar.getTopology().getNodeTemplates();
} catch (IOException | ParsingException ex) {
throw new OrchestratorException(ex);
}
Expand Down Expand Up @@ -542,9 +568,7 @@ protected Multimap<JobDependencyType, IndigoJob> generateJobGraph(Deployment dep
}

// Update jobs hierarchy with roles to create the graph
for (

Map.Entry<String, IndigoJob> job : jobs.entrySet()) {
for (Map.Entry<String, IndigoJob> job : jobs.entrySet()) {
IndigoJob indigoJob = job.getValue();

// Switch node type
Expand All @@ -569,6 +593,67 @@ protected Multimap<JobDependencyType, IndigoJob> generateJobGraph(Deployment dep
return jobGraph;
}

/**
* TEMPORARY method to replace hardcoded INDIGO params in TOSCA template (i.e. OneData) string.
*
* @param template
* the string TOSCA template.
* @param od
* the OneData settings.
*/
protected String replaceHardCodedParams(String template, OneData od) {

// Replace OneData properties
template = template.replace("TOKEN_TO_BE_SET_BY_THE_ORCHESTRATOR", od.getToken());
template = template.replace("DATA_SPACE_TO_BE_SET_BY_THE_ORCHESTRATOR", od.getSpace());
template = template.replace("PATH_TO_BE_SET_BY_THE_ORCHESTRATOR", od.getPath());
template =
template.replace("ONEDATA_PROVIDERS_TO_BE_SET_BY_THE_ORCHESTRATOR", od.getProvider());

return template;
}

public static class OneData {
private String token;
private String space;
private String path;
private String provider;

public OneData(String token, String space, String path, String provider) {
super();
this.token = token;
this.space = space;
this.path = path;
this.provider = provider;
}

public String getToken() {
return token;
}

public String getSpace() {
return space;
}

public String getPath() {
return path;
}

public String getProvider() {
return provider;
}

}

protected void putStringProperty(NodeTemplate nodeTemplate, String name, String value) {
ScalarPropertyValue scalarPropertyValue = new ScalarPropertyValue(value);
scalarPropertyValue.setPrintable(true);
if (nodeTemplate.getProperties() == null) {
nodeTemplate.setProperties(new HashMap<>());
}
nodeTemplate.getProperties().put(name, scalarPropertyValue);
}

protected boolean isChronosNode(NodeTemplate nodeTemplate) {
return nodeTemplate.getType().equals("tosca.nodes.indigo.Container.Application.Docker.Chronos");
}
Expand All @@ -582,7 +667,6 @@ protected List<String> getJobParents(NodeTemplate nodeTemplate, String nodeName,
return nodeName.equals("chronos_job_upload") ? Lists.newArrayList("chronos_job") : null;
}

@SuppressWarnings("unchecked")
protected Job createJob(Map<String, NodeTemplate> nodes, String deploymentId, String nodeName,
NodeTemplate nodeTemplate) {
try {
Expand Down Expand Up @@ -631,12 +715,14 @@ protected Job createJob(Map<String, NodeTemplate> nodes, String deploymentId, St
// Docker image
// TODO Remove hard-coded?
String supportedType = "tosca.artifacts.Deployment.Image.Container.Docker";
DeploymentArtifact image = nodeTemplate.getArtifacts().get("image");
DeploymentArtifact image;
// <image> artifact available
if (image == null)
if (nodeTemplate.getArtifacts() == null
|| (image = nodeTemplate.getArtifacts().get("image")) == null) {
throw new IllegalArgumentException(
String.format("<image> artifact not found in node <%s> of type <%s>", nodeName,
nodeTemplate.getType()));
}

// <image> artifact type check
if (!image.getArtifactType().equals(supportedType)) {
Expand Down Expand Up @@ -672,25 +758,33 @@ protected Job createJob(Map<String, NodeTemplate> nodes, String deploymentId, St
Container container = new Container();
container.setType("DOCKER");
container.setImage((String) nodeTemplate.getArtifacts().get("image").getFile());
if (dockerNumCpus != null)
if (dockerNumCpus != null) {
chronosJob.setCpus(dockerNumCpus);
if (dockerMemSize != null)
}
if (dockerMemSize != null) {
chronosJob.setMem(dockerMemSize);
}

chronosJob.setContainer(container);

return chronosJob;
} catch (Exception ex) {
throw new RuntimeException(
String.format("Failed to parse node <%s> of type <%s>", nodeName, nodeTemplate.getType()),
ex);
throw new RuntimeException(String.format("Failed to parse node <%s> of type <%s>: %s",
nodeName, nodeTemplate.getType(), ex.getMessage()), ex);
}
}

public enum JobState {
FRESH, FAILURE, SUCCESS;
}

/**
* Computes the Chronos job's state based on current success and error count.
*
* @param job
* the {@link Job}.
* @return the {@link JobState}.
*/
public static JobState getLastState(Job job) {
// State = Fresh (success + error = 0), Success (success > 0), Failure (error > 0)
// NOTE that Chronos increments the error only after all the retries has failed (x retries -> +1
Expand Down
4 changes: 4 additions & 0 deletions src/main/resources/onedata.properties
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
token=<token-example>
space=<space-example>
path=<path-example>
provider=<provider-example>
44 changes: 28 additions & 16 deletions src/test/resources/tosca/chronos_tosca_minimal.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -25,10 +25,14 @@ topology_template:
description: 'Execute app'
#command: /bin/bash env #fails!
command: env
uris: ['http://a.it','http://b.it']
uris: []
retries: 3
environment_variables:
INPUT_ONEDATA_SPACE: 'tbd'
#OUTPUT_FILENAMES: { get_input: output_filenames }
ONEDATA_SERVICE_TOKEN: 'TOKEN_TO_BE_SET_BY_THE_ORCHESTRATOR'
ONEDATA_SPACE: 'DATA_SPACE_TO_BE_SET_BY_THE_ORCHESTRATOR'
ONEDATA_PATH: 'PATH_TO_BE_SET_BY_THE_ORCHESTRATOR'
ONEDATA_PROVIDERS: 'ONEDATA_PROVIDERS_TO_BE_SET_BY_THE_ORCHESTRATOR'
artifacts:
image:
file: libmesos/ubuntu
Expand All @@ -45,17 +49,25 @@ topology_template:
mem_size: 512 MB

chronos_job_upload:
type: tosca.nodes.indigo.Container.Application.Docker.Chronos
properties:
description: 'Upload output data'
command: '/bin/bash run.sh'
retries: 3
environment_variables:
ONEDATA_SERVICE_TOKEN: 'TOKEN_TO_BE_SET_BY_THE_ORCHESTRATOR'
ONEDATA_SPACE: 'DATA_SPACE_TO_BE_SET_BY_THE_ORCHESTRATOR'
ONEDATA_PATH: 'PATH_TO_BE_SET_BY_THE_ORCHESTRATOR'
ONEDATA_PROVIDERS: 'ONEDATA_PROVIDERS_TO_BE_SET_BY_THE_ORCHESTRATOR'
artifacts:
image:
file: indigodatacloud/jobuploader
type: tosca.artifacts.Deployment.Image.Container.Docker
type: tosca.nodes.indigo.Container.Application.Docker.Chronos
properties:
description: 'Upload output data'
command: '/bin/bash run.sh'
retries: 3
environment_variables:
#OUTPUT_PROTOCOL: { get_input: output_protocol }
#OUTPUT_ENDPOINT: { get_input: output_endpoint }
#OUTPUT_PATH: { get_input: output_path }
#OUTPUT_USERNAME: { get_input: output_username }
#OUTPUT_PASSWORD: { get_input: output_password }
#OUTPUT_TENANT: { get_input: output_tenant }
#OUTPUT_REGION: { get_input: output_region }
#OUTPUT_FILENAMES: { get_input: output_filenames }
ONEDATA_SERVICE_TOKEN: 'TOKEN_TO_BE_SET_BY_THE_ORCHESTRATOR'
ONEDATA_SPACE: 'DATA_SPACE_TO_BE_SET_BY_THE_ORCHESTRATOR'
ONEDATA_PATH: 'PATH_TO_BE_SET_BY_THE_ORCHESTRATOR'
ONEDATA_PROVIDERS: 'ONEDATA_PROVIDERS_TO_BE_SET_BY_THE_ORCHESTRATOR'
artifacts:
image:
file: indigodatacloud/jobuploader
type: tosca.artifacts.Deployment.Image.Container.Docker

0 comments on commit 8e3c6c8

Please sign in to comment.